From c30980a9f4cd17ee090746afb8af6f7bd5c8397e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:58:42 +0900 Subject: [PATCH 01/28] ci: use Ubuntu 22.04 (#3083) Update the CI configuration and start testing on Ubuntu 22.04. Fixes #3084 --- .bazelci/presubmit.yml | 64 ++++++++++++++-------------- CHANGELOG.md | 1 + WORKSPACE | 2 +- python/private/internal_dev_deps.bzl | 2 +- 4 files changed, 35 insertions(+), 34 deletions(-) diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 07ffa4eaac..6457363ccd 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -91,20 +91,20 @@ tasks: <<: *common_workspace_flags_min_bazel <<: *minimum_supported_version name: "Gazelle: workspace, minimum supported Bazel version" - platform: ubuntu2004 + platform: ubuntu2204 build_targets: ["//..."] test_targets: ["//..."] working_directory: gazelle gazelle_extension_workspace: <<: *common_workspace_flags name: "Gazelle: workspace" - platform: ubuntu2004 + platform: ubuntu2204 build_targets: ["//..."] test_targets: ["//..."] working_directory: gazelle gazelle_extension: name: "Gazelle: default settings" - platform: ubuntu2004 + platform: ubuntu2204 build_targets: ["//..."] test_targets: ["//..."] working_directory: gazelle @@ -114,28 +114,28 @@ tasks: <<: *reusable_config <<: *common_workspace_flags_min_bazel name: "Default: Ubuntu, workspace, minimum Bazel" - platform: ubuntu2004 + platform: ubuntu2204 ubuntu_min_bzlmod: <<: *minimum_supported_version <<: *reusable_config name: "Default: Ubuntu, bzlmod, minimum Bazel" - platform: ubuntu2004 + platform: ubuntu2204 bazel: 7.x ubuntu: <<: *reusable_config name: "Default: Ubuntu" - platform: ubuntu2004 + platform: ubuntu2204 ubuntu_upcoming: <<: *reusable_config name: "Default: Ubuntu, upcoming Bazel" - platform: ubuntu2004 + platform: ubuntu2204 bazel: last_rc ubuntu_workspace: <<: *reusable_config <<: *common_workspace_flags name: "Default: Ubuntu, workspace" - platform: ubuntu2004 + platform: ubuntu2204 mac_workspace: <<: *reusable_config <<: *common_workspace_flags @@ -185,7 +185,7 @@ tasks: <<: *minimum_supported_version <<: *reusable_config name: "RBE: Ubuntu, minimum Bazel" - platform: rbe_ubuntu2004 + platform: rbe_ubuntu2204 build_flags: # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, # which prevents cc toolchain autodetection from working correctly @@ -203,7 +203,7 @@ tasks: rbe: <<: *reusable_config name: "RBE: Ubuntu" - platform: rbe_ubuntu2004 + platform: rbe_ubuntu2204 # TODO @aignas 2024-12-11: get the RBE working in CI for bazel 8.0 # See https://github.com/bazelbuild/rules_python/issues/2499 bazel: 7.x @@ -217,13 +217,13 @@ tasks: <<: *common_workspace_flags_min_bazel name: "examples/build_file_generation: Ubuntu, workspace, minimum Bazel" working_directory: examples/build_file_generation - platform: ubuntu2004 + platform: ubuntu2204 integration_test_build_file_generation_ubuntu_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags name: "examples/build_file_generation: Ubuntu, workspace" working_directory: examples/build_file_generation - platform: ubuntu2004 + platform: ubuntu2204 integration_test_build_file_generation_debian_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -249,21 +249,21 @@ tasks: coverage_targets: ["//:test"] name: "examples/bzlmod: Ubuntu, minimum Bazel" working_directory: examples/bzlmod - platform: ubuntu2004 + platform: ubuntu2204 bazel: 7.x integration_test_bzlmod_ubuntu: <<: *reusable_build_test_all <<: *coverage_targets_example_bzlmod name: "examples/bzlmod: Ubuntu" working_directory: examples/bzlmod - platform: ubuntu2004 + platform: ubuntu2204 bazel: 7.x integration_test_bzlmod_ubuntu_upcoming: <<: *reusable_build_test_all <<: *coverage_targets_example_bzlmod name: "examples/bzlmod: Ubuntu, upcoming Bazel" working_directory: examples/bzlmod - platform: ubuntu2004 + platform: ubuntu2204 bazel: last_rc integration_test_bzlmod_debian: <<: *reusable_build_test_all @@ -276,7 +276,7 @@ tasks: <<: *reusable_build_test_all name: "examples/bzlmod: bazel vendor" working_directory: examples/bzlmod - platform: ubuntu2004 + platform: ubuntu2204 shell_commands: - "bazel vendor --vendor_dir=./vendor //..." - "bazel build --vendor_dir=./vendor //..." @@ -316,19 +316,19 @@ tasks: <<: *coverage_targets_example_bzlmod_build_file_generation name: "examples/bzlmod_build_file_generation: Ubuntu, minimum Bazel" working_directory: examples/bzlmod_build_file_generation - platform: ubuntu2004 + platform: ubuntu2204 bazel: 7.x integration_test_bzlmod_generation_build_files_ubuntu: <<: *reusable_build_test_all <<: *coverage_targets_example_bzlmod_build_file_generation name: "examples/bzlmod_build_file_generation: Ubuntu" working_directory: examples/bzlmod_build_file_generation - platform: ubuntu2004 + platform: ubuntu2204 integration_test_bzlmod_generation_build_files_ubuntu_run: <<: *reusable_build_test_all name: "examples/bzlmod_build_file_generation: Ubuntu, Gazelle and pip" working_directory: examples/bzlmod_build_file_generation - platform: ubuntu2004 + platform: ubuntu2204 shell_commands: - "bazel run //:gazelle_python_manifest.update" - "bazel run //:gazelle -- update" @@ -357,7 +357,7 @@ tasks: <<: *coverage_targets_example_multi_python name: "examples/multi_python_versions: Ubuntu, workspace" working_directory: examples/multi_python_versions - platform: ubuntu2004 + platform: ubuntu2204 integration_test_multi_python_versions_debian_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -386,19 +386,19 @@ tasks: <<: *reusable_build_test_all name: "examples/pip_parse: Ubuntu, workspace, minimum supported Bazel version" working_directory: examples/pip_parse - platform: ubuntu2004 + platform: ubuntu2204 integration_test_pip_parse_ubuntu_min_bzlmod: <<: *minimum_supported_version <<: *reusable_build_test_all name: "examples/pip_parse: Ubuntu, bzlmod, minimum supported Bazel version" working_directory: examples/pip_parse - platform: ubuntu2004 + platform: ubuntu2204 bazel: 7.x integration_test_pip_parse_ubuntu: <<: *reusable_build_test_all name: "examples/pip_parse: Ubuntu" working_directory: examples/pip_parse - platform: ubuntu2004 + platform: ubuntu2204 integration_test_pip_parse_debian: <<: *reusable_build_test_all name: "examples/pip_parse: Debian" @@ -421,13 +421,13 @@ tasks: <<: *reusable_build_test_all name: "examples/pip_parse_vendored: Ubuntu, workspace, minimum Bazel" working_directory: examples/pip_parse_vendored - platform: ubuntu2004 + platform: ubuntu2204 integration_test_pip_parse_vendored_ubuntu: <<: *reusable_build_test_all <<: *common_workspace_flags name: "examples/pip_parse_vendored: Ubuntu" working_directory: examples/pip_parse_vendored - platform: ubuntu2004 + platform: ubuntu2204 integration_test_pip_parse_vendored_debian: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -450,7 +450,7 @@ tasks: <<: *common_workspace_flags name: "examples/py_proto_library: Ubuntu, workspace" working_directory: examples/py_proto_library - platform: ubuntu2004 + platform: ubuntu2204 integration_test_py_proto_library_debian_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -475,7 +475,7 @@ tasks: <<: *common_workspace_flags name: "examples/pip_repository_annotations: Ubuntu, workspace" working_directory: examples/pip_repository_annotations - platform: ubuntu2004 + platform: ubuntu2204 integration_test_pip_repository_annotations_debian_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags @@ -498,7 +498,7 @@ tasks: integration_test_bazelinbazel_ubuntu: <<: *common_bazelinbazel_config name: "tests/integration bazel-in-bazel: Ubuntu" - platform: ubuntu2004 + platform: ubuntu2204 integration_test_bazelinbazel_debian: <<: *common_bazelinbazel_config name: "tests/integration bazel-in-bazel: Debian" @@ -508,7 +508,7 @@ tasks: <<: *reusable_build_test_all name: "compile_pip_requirements: Ubuntu" working_directory: tests/integration/compile_pip_requirements - platform: ubuntu2004 + platform: ubuntu2204 shell_commands: # Make a change to the locked requirements and then assert that //:requirements.update does the # right thing. @@ -596,7 +596,7 @@ tasks: <<: *common_workspace_flags_min_bazel name: "compile_pip_requirements_test_from_external_repo: Ubuntu, workspace, minimum Bazel" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: ubuntu2004 + platform: ubuntu2204 shell_commands: # Assert that @compile_pip_requirements//:requirements_test does the right thing. - "bazel test @compile_pip_requirements//..." @@ -604,7 +604,7 @@ tasks: <<: *minimum_supported_version name: "compile_pip_requirements_test_from_external_repo: Ubuntu, bzlmod, minimum Bazel" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: ubuntu2004 + platform: ubuntu2204 bazel: 7.x shell_commands: # Assert that @compile_pip_requirements//:requirements_test does the right thing. @@ -612,7 +612,7 @@ tasks: integration_compile_pip_requirements_test_from_external_repo_ubuntu: name: "compile_pip_requirements_test_from_external_repo: Ubuntu" working_directory: tests/integration/compile_pip_requirements_test_from_external_repo - platform: ubuntu2004 + platform: ubuntu2204 shell_commands: # Assert that @compile_pip_requirements//:requirements_test does the right thing. - "bazel test @compile_pip_requirements//..." diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f02c8bbb4..ad68669df2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ END_UNRELEASED_TEMPLATE * (toolchain) Python 3.13 now references 3.13.5 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) +* (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations. {#v0-0-0-fixed} ### Fixed diff --git a/WORKSPACE b/WORKSPACE index dddc5105ed..5c2136666d 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -95,7 +95,7 @@ load("@bazelci_rules//:rbe_repo.bzl", "rbe_preconfig") # otherwise refer to RBE docs. rbe_preconfig( name = "buildkite_config", - toolchain = "ubuntu1804-bazel-java11", + toolchain = "ubuntu2204", ) local_repository( diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index ca34dc698a..d621a5d941 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -26,7 +26,7 @@ def _internal_dev_deps_impl(mctx): # otherwise refer to RBE docs. rbe_preconfig( name = "buildkite_config", - toolchain = "ubuntu1804-bazel-java11", + toolchain = "ubuntu2204", ) runtime_env_repo(name = "rules_python_runtime_env_tc_info") From 6f27511a35baa7d0de302e504cb161cf0af2f2bf Mon Sep 17 00:00:00 2001 From: yushan26 <107004874+yushan26@users.noreply.github.com> Date: Mon, 14 Jul 2025 16:51:42 -0700 Subject: [PATCH 02/28] fix(gazelle) Update gazelle to properly process multi-line python imports (#3077) A python import may be imported as: ``` from foo.bar.application.\ pipeline.model import ( Baz ) ``` However, gazelle fails to resolve this import with the error: `line 30: "foo.bar.application.pipeline.model\\\n pipeline.mode.Baz" is an invalid dependency:` Clean up the imports such that whitespace and \n are removed from the import path. --------- Co-authored-by: yushan Co-authored-by: Douglas Thor --- CHANGELOG.md | 1 + gazelle/python/file_parser.go | 14 +++ gazelle/python/file_parser_test.go | 92 +++++++++++++++++++ .../import_nested_var/__init__.py | 6 +- 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ad68669df2..834a2c1a39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ END_UNRELEASED_TEMPLATE ({gh-issue}`3043`). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. +* Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle. {#v0-0-0-added} ### Added diff --git a/gazelle/python/file_parser.go b/gazelle/python/file_parser.go index 31fce02712..e129337e11 100644 --- a/gazelle/python/file_parser.go +++ b/gazelle/python/file_parser.go @@ -144,6 +144,16 @@ func parseImportStatement(node *sitter.Node, code []byte) (Module, bool) { return Module{}, false } +// cleanImportString removes backslashes and all whitespace from the string. +func cleanImportString(s string) string { + s = strings.ReplaceAll(s, "\r\n", "") + s = strings.ReplaceAll(s, "\\", "") + s = strings.ReplaceAll(s, " ", "") + s = strings.ReplaceAll(s, "\n", "") + s = strings.ReplaceAll(s, "\t", "") + return s +} + // parseImportStatements parses a node for import statements, returning true if the node is // an import statement. It updates FileParser.output.Modules with the `module` that the // import represents. @@ -154,6 +164,8 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { if !ok { continue } + m.From = cleanImportString(m.From) + m.Name = cleanImportString(m.Name) m.Filepath = p.relFilepath m.TypeCheckingOnly = p.inTypeCheckingBlock if strings.HasPrefix(m.Name, ".") { @@ -163,6 +175,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { } } else if node.Type() == sitterNodeTypeImportFromStatement { from := node.Child(1).Content(p.code) + from = cleanImportString(from) // If the import is from the current package, we don't need to add it to the modules i.e. from . import Class1. // If the import is from a different relative package i.e. from .package1 import foo, we need to add it to the modules. if from == "." { @@ -175,6 +188,7 @@ func (p *FileParser) parseImportStatements(node *sitter.Node) bool { } m.Filepath = p.relFilepath m.From = from + m.Name = cleanImportString(m.Name) m.Name = fmt.Sprintf("%s.%s", from, m.Name) m.TypeCheckingOnly = p.inTypeCheckingBlock p.output.Modules = append(p.output.Modules, m) diff --git a/gazelle/python/file_parser_test.go b/gazelle/python/file_parser_test.go index f4db1a316b..0a6fd1b4ab 100644 --- a/gazelle/python/file_parser_test.go +++ b/gazelle/python/file_parser_test.go @@ -291,3 +291,95 @@ def example_function(): } } } + +func TestParseImportStatements_MultilineWithBackslashAndWhitespace(t *testing.T) { + t.Parallel() + t.Run("multiline from import", func(t *testing.T) { + p := NewFileParser() + code := []byte(`from foo.bar.\ + baz import ( + Something, + AnotherThing +) + +from foo\ + .test import ( + Foo, + Bar +) +`) + p.SetCodeAndFile(code, "", "test.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + // Updated expected to match parser output + expected := []Module{ + { + Name: "foo.bar.baz.Something", + LineNumber: 3, + Filepath: "test.py", + From: "foo.bar.baz", + }, + { + Name: "foo.bar.baz.AnotherThing", + LineNumber: 4, + Filepath: "test.py", + From: "foo.bar.baz", + }, + { + Name: "foo.test.Foo", + LineNumber: 9, + Filepath: "test.py", + From: "foo.test", + }, + { + Name: "foo.test.Bar", + LineNumber: 10, + Filepath: "test.py", + From: "foo.test", + }, + } + assert.ElementsMatch(t, expected, output.Modules) + }) + t.Run("multiline import", func(t *testing.T) { + p := NewFileParser() + code := []byte(`import foo.bar.\ + baz +`) + p.SetCodeAndFile(code, "", "test.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + // Updated expected to match parser output + expected := []Module{ + { + Name: "foo.bar.baz", + LineNumber: 1, + Filepath: "test.py", + From: "", + }, + } + assert.ElementsMatch(t, expected, output.Modules) + }) + t.Run("windows line endings", func(t *testing.T) { + p := NewFileParser() + code := []byte("from foo.bar.\r\n baz import (\r\n Something,\r\n AnotherThing\r\n)\r\n") + p.SetCodeAndFile(code, "", "test.py") + output, err := p.Parse(context.Background()) + assert.NoError(t, err) + // Updated expected to match parser output + expected := []Module{ + { + Name: "foo.bar.baz.Something", + LineNumber: 3, + Filepath: "test.py", + From: "foo.bar.baz", + }, + { + Name: "foo.bar.baz.AnotherThing", + LineNumber: 4, + Filepath: "test.py", + From: "foo.bar.baz", + }, + } + assert.ElementsMatch(t, expected, output.Modules) + }) +} diff --git a/gazelle/python/testdata/from_imports/import_nested_var/__init__.py b/gazelle/python/testdata/from_imports/import_nested_var/__init__.py index d0f51c443c..20eda530e5 100644 --- a/gazelle/python/testdata/from_imports/import_nested_var/__init__.py +++ b/gazelle/python/testdata/from_imports/import_nested_var/__init__.py @@ -13,4 +13,8 @@ # limitations under the License. # baz is a variable in foo/bar/baz.py -from foo.bar.baz import baz +from foo\ + .bar.\ + baz import ( + baz + ) From dd6550f18477105f362e44cb4868d9007e340d83 Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Mon, 14 Jul 2025 19:57:52 -0400 Subject: [PATCH 03/28] feat(gazelle): Gazelle plugin generates py_proto_library (#3057) Fixes https://github.com/bazel-contrib/rules_python/issues/2994. Please go over this with a fine-toothed comb! This is my first contribution to `rules_python` / the gazelle plugin, and while I've worked in Gazelle before, I'm pretty unfamiliar with the Python plugin's architecture. This adds support in the Gazelle plugin for generating `py_proto_library` rules automatically, if there are any `proto_library` rules detected in a given package. We do this via a new Gazelle directive, `python_generate_proto`, which defaults to `true`, and controls whether these rules are generated. See the tests in `testdata/directive_python_generate_proto` for examples. By default, we source the `py_proto_library` rule from the `@protobuf` repository. I think this the intended long-term home of the rule? Users are expected to use `gazelle:map_kind` to change this if need be. I haven't done anything here to support resolution of imports of `py_proto_library`. I think this is worth landing first, to save folks from having to maintain these by hand. But this should lay the foundation for resolving that in https://github.com/bazel-contrib/rules_python/issues/1703. --------- Co-authored-by: Douglas Thor --- CHANGELOG.md | 2 + examples/bzlmod/py_proto_library/BUILD.bazel | 4 +- .../example.com/another_proto/BUILD.bazel | 2 +- .../example.com/proto/BUILD.bazel | 2 +- examples/py_proto_library/BUILD.bazel | 4 +- .../example.com/another_proto/BUILD.bazel | 2 +- .../example.com/proto/BUILD.bazel | 2 +- gazelle/README.md | 37 ++++++++++++ gazelle/python/BUILD.bazel | 6 +- gazelle/python/configure.go | 7 +++ gazelle/python/generate.go | 52 ++++++++++++++++ gazelle/python/kinds.go | 60 +++++++++++++------ .../directive_python_generate_proto/README.md | 9 +++ .../directive_python_generate_proto/WORKSPACE | 1 + .../directive_python_generate_proto/test.yaml | 3 + .../test1_default_with_proto/BUILD.in | 9 +++ .../test1_default_with_proto/BUILD.out | 9 +++ .../test1_default_with_proto/foo.proto | 7 +++ .../test2_default_without_proto/BUILD.in | 1 + .../test2_default_without_proto/BUILD.out | 1 + .../test3_disabled_with_proto/BUILD.in | 9 +++ .../test3_disabled_with_proto/BUILD.out | 9 +++ .../test3_disabled_with_proto/foo.proto | 7 +++ .../test4_disabled_without_proto/BUILD.in | 1 + .../test4_disabled_without_proto/BUILD.out | 1 + .../test5_enabled_with_proto/BUILD.in | 9 +++ .../test5_enabled_with_proto/BUILD.out | 16 +++++ .../test5_enabled_with_proto/foo.proto | 7 +++ .../test6_enabled_without_proto/BUILD.in | 1 + .../test6_enabled_without_proto/BUILD.out | 1 + .../test7_removes_when_unnecessary/BUILD.in | 16 +++++ .../test7_removes_when_unnecessary/BUILD.out | 1 + .../BUILD.in | 16 +++++ .../BUILD.out | 16 +++++ .../foo.proto | 7 +++ .../BUILD.in | 9 +++ .../BUILD.out | 16 +++++ .../MODULE.bazel | 1 + .../README.md | 6 ++ .../WORKSPACE | 0 .../foo.proto | 7 +++ .../test.yaml | 3 + .../BUILD.in | 9 +++ .../BUILD.out | 16 +++++ .../MODULE.bazel | 1 + .../README.md | 7 +++ .../WORKSPACE | 0 .../foo.proto | 7 +++ .../test.yaml | 3 + gazelle/pythonconfig/pythonconfig.go | 16 +++++ 50 files changed, 412 insertions(+), 26 deletions(-) create mode 100644 gazelle/python/testdata/directive_python_generate_proto/README.md create mode 100644 gazelle/python/testdata/directive_python_generate_proto/WORKSPACE create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test.yaml create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/foo.proto create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/foo.proto create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/foo.proto create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/foo.proto create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/MODULE.bazel create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/README.md create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/WORKSPACE create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/foo.proto create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/test.yaml create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/MODULE.bazel create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/README.md create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/WORKSPACE create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/foo.proto create mode 100644 gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 834a2c1a39..e74f14b1db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,8 @@ END_UNRELEASED_TEMPLATE * 3.12.11 * 3.13.5 * 3.14.0b3 +* (gazelle) New directive `gazelle:python_generate_proto`; when `true`, + Gazelle generates `py_proto_library` rules for `proto_library`. `false` by default. {#v0-0-0-removed} ### Removed diff --git a/examples/bzlmod/py_proto_library/BUILD.bazel b/examples/bzlmod/py_proto_library/BUILD.bazel index 969cb8e9f7..daea410365 100644 --- a/examples/bzlmod/py_proto_library/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/BUILD.bazel @@ -6,7 +6,7 @@ py_test( srcs = ["test.py"], main = "test.py", deps = [ - "//py_proto_library/example.com/proto:pricetag_proto_py_pb2", + "//py_proto_library/example.com/proto:pricetag_py_pb2", ], ) @@ -14,7 +14,7 @@ py_test( name = "message_test", srcs = ["message_test.py"], deps = [ - "//py_proto_library/example.com/another_proto:message_proto_py_pb2", + "//py_proto_library/example.com/another_proto:message_py_pb2", ], ) diff --git a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel index 785d90d01e..29f08c21ca 100644 --- a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel @@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( - name = "message_proto_py_pb2", + name = "message_py_pb2", visibility = ["//visibility:public"], deps = [":message_proto"], ) diff --git a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel index 72af672219..1f8e8f2818 100644 --- a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel @@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( - name = "pricetag_proto_py_pb2", + name = "pricetag_py_pb2", visibility = ["//visibility:public"], deps = [":pricetag_proto"], ) diff --git a/examples/py_proto_library/BUILD.bazel b/examples/py_proto_library/BUILD.bazel index d782fb296d..b57c528511 100644 --- a/examples/py_proto_library/BUILD.bazel +++ b/examples/py_proto_library/BUILD.bazel @@ -5,7 +5,7 @@ py_test( srcs = ["test.py"], main = "test.py", deps = [ - "//example.com/proto:pricetag_proto_py_pb2", + "//example.com/proto:pricetag_py_pb2", ], ) @@ -13,6 +13,6 @@ py_test( name = "message_test", srcs = ["message_test.py"], deps = [ - "//example.com/another_proto:message_proto_py_pb2", + "//example.com/another_proto:message_py_pb2", ], ) diff --git a/examples/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/py_proto_library/example.com/another_proto/BUILD.bazel index 3d841554e9..55e83a209a 100644 --- a/examples/py_proto_library/example.com/another_proto/BUILD.bazel +++ b/examples/py_proto_library/example.com/another_proto/BUILD.bazel @@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( - name = "message_proto_py_pb2", + name = "message_py_pb2", visibility = ["//visibility:public"], deps = [":message_proto"], ) diff --git a/examples/py_proto_library/example.com/proto/BUILD.bazel b/examples/py_proto_library/example.com/proto/BUILD.bazel index f84454f531..fdf2e6fe32 100644 --- a/examples/py_proto_library/example.com/proto/BUILD.bazel +++ b/examples/py_proto_library/example.com/proto/BUILD.bazel @@ -2,7 +2,7 @@ load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( - name = "pricetag_proto_py_pb2", + name = "pricetag_py_pb2", visibility = ["//visibility:public"], deps = [":pricetag_proto"], ) diff --git a/gazelle/README.md b/gazelle/README.md index 3dc8e12a0a..35a1e4f701 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -224,6 +224,8 @@ Python-specific directives are as follows: | Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".| | `# gazelle:python_generate_pyi_deps` | `false` | | Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. | +| [`# gazelle:python_generate_proto`](#directive-python_generate_proto) | `false` | +| Controls whether to generate a `py_proto_library` for each `proto_library` in the package. By default we load this rule from the `@protobuf` repository; use `gazelle:map_kind` if you need to load this from somewhere else. | #### Directive: `python_root`: @@ -484,6 +486,41 @@ def py_test(name, main=None, **kwargs): ) ``` +#### Directive: `python_generate_proto`: + +When `# gazelle:python_generate_proto true`, Gazelle will generate one +`py_proto_library` for each `proto_library`, generating Python clients for +protobuf in each package. By default this is turned off. Gazelle will also +generate a load statement for the `py_proto_library` - attempting to detect +the configured name for the `@protobuf` / `@com_google_protobuf` repo in your +`MODULE.bazel`, and otherwise falling back to `@com_google_protobuf` for +compatibility with `WORKSPACE`. + +For example, in a package with `# gazelle:python_generate_proto true` and a +`foo.proto`, if you have both the proto extension and the Python extension +loaded into Gazelle, you'll get something like: + +```starlark +load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) +``` + +When `false`, Gazelle will ignore any `py_proto_library`, including previously-generated or hand-created rules. + ### Annotations *Annotations* refer to comments found _within Python files_ that configure how diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 8e8216ddd4..1a7c54f4b2 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -34,6 +34,7 @@ go_library( "@bazel_gazelle//config:go_default_library", "@bazel_gazelle//label:go_default_library", "@bazel_gazelle//language:go_default_library", + "@bazel_gazelle//language/proto:go_default_library", "@bazel_gazelle//repo:go_default_library", "@bazel_gazelle//resolve:go_default_library", "@bazel_gazelle//rule:go_default_library", @@ -91,7 +92,10 @@ gazelle_test( gazelle_binary( name = "gazelle_binary", - languages = [":python"], + languages = [ + "@bazel_gazelle//language/proto", + ":python", + ], visibility = ["//visibility:public"], ) diff --git a/gazelle/python/configure.go b/gazelle/python/configure.go index db80fc1a22..7131be283d 100644 --- a/gazelle/python/configure.go +++ b/gazelle/python/configure.go @@ -70,6 +70,7 @@ func (py *Configurer) KnownDirectives() []string { pythonconfig.LabelNormalization, pythonconfig.GeneratePyiDeps, pythonconfig.ExperimentalAllowRelativeImports, + pythonconfig.GenerateProto, } } @@ -237,6 +238,12 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) { log.Fatal(err) } config.SetGeneratePyiDeps(v) + case pythonconfig.GenerateProto: + v, err := strconv.ParseBool(strings.TrimSpace(d.Value)) + if err != nil { + log.Fatal(err) + } + config.SetGenerateProto(v) } } diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index c1edec4731..343743559f 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -226,6 +226,10 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes var result language.GenerateResult result.Gen = make([]*rule.Rule, 0) + if cfg.GenerateProto() { + generateProtoLibraries(args, pythonProjectRoot, visibility, &result) + } + collisionErrors := singlylinkedlist.New() appendPyLibrary := func(srcs *treeset.Set, pyLibraryTargetName string) { @@ -551,3 +555,51 @@ func ensureNoCollision(file *rule.File, targetName, kind string) error { } return nil } + +func generateProtoLibraries(args language.GenerateArgs, pythonProjectRoot string, visibility []string, res *language.GenerateResult) { + // First, enumerate all the proto_library in this package. + var protoRuleNames []string + for _, r := range args.OtherGen { + if r.Kind() != "proto_library" { + continue + } + protoRuleNames = append(protoRuleNames, r.Name()) + } + sort.Strings(protoRuleNames) + + // Next, enumerate all the pre-existing py_proto_library in this package, so we can delete unnecessary rules later. + pyProtoRules := map[string]bool{} + if args.File != nil { + for _, r := range args.File.Rules { + if r.Kind() == "py_proto_library" { + pyProtoRules[r.Name()] = false + } + } + } + + emptySiblings := treeset.Set{} + // Generate a py_proto_library for each proto_library. + for _, protoRuleName := range protoRuleNames { + pyProtoLibraryName := strings.TrimSuffix(protoRuleName, "_proto") + "_py_pb2" + pyProtoLibrary := newTargetBuilder(pyProtoLibraryKind, pyProtoLibraryName, pythonProjectRoot, args.Rel, &emptySiblings). + addVisibility(visibility). + addResolvedDependency(":" + protoRuleName). + generateImportsAttribute().build() + + res.Gen = append(res.Gen, pyProtoLibrary) + res.Imports = append(res.Imports, pyProtoLibrary.PrivateAttr(config.GazelleImportsKey)) + pyProtoRules[pyProtoLibrary.Name()] = true + + } + + // Finally, emit an empty rule for each pre-existing py_proto_library that we didn't already generate. + for ruleName, generated := range pyProtoRules { + if generated { + continue + } + + emptyRule := newTargetBuilder(pyProtoLibraryKind, ruleName, pythonProjectRoot, args.Rel, &emptySiblings).build() + res.Empty = append(res.Empty, emptyRule) + } + +} diff --git a/gazelle/python/kinds.go b/gazelle/python/kinds.go index ff3f6ce829..a4ce572aaa 100644 --- a/gazelle/python/kinds.go +++ b/gazelle/python/kinds.go @@ -15,13 +15,16 @@ package python import ( + "fmt" + "github.com/bazelbuild/bazel-gazelle/rule" ) const ( - pyBinaryKind = "py_binary" - pyLibraryKind = "py_library" - pyTestKind = "py_test" + pyBinaryKind = "py_binary" + pyLibraryKind = "py_library" + pyProtoLibraryKind = "py_proto_library" + pyTestKind = "py_test" ) // Kinds returns a map that maps rule names (kinds) and information on how to @@ -32,7 +35,7 @@ func (*Python) Kinds() map[string]rule.KindInfo { var pyKinds = map[string]rule.KindInfo{ pyBinaryKind: { - MatchAny: false, + MatchAny: false, MatchAttrs: []string{"srcs"}, NonEmptyAttrs: map[string]bool{ "deps": true, @@ -45,7 +48,7 @@ var pyKinds = map[string]rule.KindInfo{ "srcs": true, }, ResolveAttrs: map[string]bool{ - "deps": true, + "deps": true, "pyi_deps": true, }, }, @@ -62,10 +65,16 @@ var pyKinds = map[string]rule.KindInfo{ "srcs": true, }, ResolveAttrs: map[string]bool{ - "deps": true, + "deps": true, "pyi_deps": true, }, }, + pyProtoLibraryKind: { + NonEmptyAttrs: map[string]bool{ + "deps": true, + }, + ResolveAttrs: map[string]bool{"deps": true}, + }, pyTestKind: { MatchAny: false, NonEmptyAttrs: map[string]bool{ @@ -79,26 +88,43 @@ var pyKinds = map[string]rule.KindInfo{ "srcs": true, }, ResolveAttrs: map[string]bool{ - "deps": true, + "deps": true, "pyi_deps": true, }, }, } +func (py *Python) Loads() []rule.LoadInfo { + panic("ApparentLoads should be called instead") +} + // Loads returns .bzl files and symbols they define. Every rule generated by // GenerateRules, now or in the past, should be loadable from one of these // files. -func (py *Python) Loads() []rule.LoadInfo { - return pyLoads +func (py *Python) ApparentLoads(moduleToApparentName func(string) string) []rule.LoadInfo { + return apparentLoads(moduleToApparentName) } -var pyLoads = []rule.LoadInfo{ - { - Name: "@rules_python//python:defs.bzl", - Symbols: []string{ - pyBinaryKind, - pyLibraryKind, - pyTestKind, +func apparentLoads(moduleToApparentName func(string) string) []rule.LoadInfo { + protobuf := moduleToApparentName("protobuf") + if protobuf == "" { + protobuf = "com_google_protobuf" + } + + return []rule.LoadInfo{ + { + Name: "@rules_python//python:defs.bzl", + Symbols: []string{ + pyBinaryKind, + pyLibraryKind, + pyTestKind, + }, }, - }, + { + Name: fmt.Sprintf("@%s//bazel:py_proto_library.bzl", protobuf), + Symbols: []string{ + pyProtoLibraryKind, + }, + }, + } } diff --git a/gazelle/python/testdata/directive_python_generate_proto/README.md b/gazelle/python/testdata/directive_python_generate_proto/README.md new file mode 100644 index 0000000000..54261f47ca --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/README.md @@ -0,0 +1,9 @@ +# Directive: `python_generate_proto` + +This test case asserts that the `# gazelle:python_generate_proto` directive +correctly: + +1. Uses the default value when `python_generate_proto` is not set. +2. Generates (or not) `py_proto_library` when `python_generate_proto` is set, based on whether a proto is present. + +[gh-2994]: https://github.com/bazel-contrib/rules_python/issues/2994 diff --git a/gazelle/python/testdata/directive_python_generate_proto/WORKSPACE b/gazelle/python/testdata/directive_python_generate_proto/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/directive_python_generate_proto/test.yaml b/gazelle/python/testdata/directive_python_generate_proto/test.yaml new file mode 100644 index 0000000000..36dd656b39 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test.yaml @@ -0,0 +1,3 @@ +--- +expect: + exit_code: 0 diff --git a/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.in new file mode 100644 index 0000000000..9784aafc17 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.in @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# python_generate_proto is not set, so py_proto_library is not generated. + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.out new file mode 100644 index 0000000000..9784aafc17 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/BUILD.out @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# python_generate_proto is not set, so py_proto_library is not generated. + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/foo.proto b/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/foo.proto new file mode 100644 index 0000000000..fe2af27aa6 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test1_default_with_proto/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.in new file mode 100644 index 0000000000..0a869d0fd5 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.in @@ -0,0 +1 @@ +# python_generate_proto is not set, so py_proto_library is not generated. diff --git a/gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.out new file mode 100644 index 0000000000..0a869d0fd5 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test2_default_without_proto/BUILD.out @@ -0,0 +1 @@ +# python_generate_proto is not set, so py_proto_library is not generated. diff --git a/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.in new file mode 100644 index 0000000000..62fd4be661 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.in @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto false + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.out new file mode 100644 index 0000000000..62fd4be661 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/BUILD.out @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto false + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/foo.proto b/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/foo.proto new file mode 100644 index 0000000000..022e29ae69 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test3_disabled_with_proto/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo.bar; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.in new file mode 100644 index 0000000000..b283b5fb51 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_generate_proto false diff --git a/gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.out new file mode 100644 index 0000000000..b283b5fb51 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test4_disabled_without_proto/BUILD.out @@ -0,0 +1 @@ +# gazelle:python_generate_proto false diff --git a/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.in new file mode 100644 index 0000000000..4713404b19 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.in @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.out new file mode 100644 index 0000000000..686252f27c --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/BUILD.out @@ -0,0 +1,16 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/foo.proto b/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/foo.proto new file mode 100644 index 0000000000..fe2af27aa6 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test5_enabled_with_proto/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.in new file mode 100644 index 0000000000..ce3eec6001 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_generate_proto true diff --git a/gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.out new file mode 100644 index 0000000000..ce3eec6001 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test6_enabled_without_proto/BUILD.out @@ -0,0 +1 @@ +# gazelle:python_generate_proto true diff --git a/gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.in new file mode 100644 index 0000000000..686252f27c --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.in @@ -0,0 +1,16 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.out new file mode 100644 index 0000000000..ce3eec6001 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test7_removes_when_unnecessary/BUILD.out @@ -0,0 +1 @@ +# gazelle:python_generate_proto true diff --git a/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.in new file mode 100644 index 0000000000..f14ed4fc2d --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.in @@ -0,0 +1,16 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto false + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.out new file mode 100644 index 0000000000..f14ed4fc2d --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/BUILD.out @@ -0,0 +1,16 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto false + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/foo.proto b/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/foo.proto new file mode 100644 index 0000000000..022e29ae69 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto/test8_disabled_ignores_py_proto_library/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo.bar; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.in new file mode 100644 index 0000000000..4713404b19 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.in @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.out new file mode 100644 index 0000000000..dab84a6777 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/BUILD.out @@ -0,0 +1,16 @@ +load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/MODULE.bazel b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/MODULE.bazel new file mode 100644 index 0000000000..66d64afe03 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/MODULE.bazel @@ -0,0 +1 @@ +bazel_dep(name = "protobuf", version = "29.3") diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/README.md b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/README.md new file mode 100644 index 0000000000..2d91ccff56 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/README.md @@ -0,0 +1,6 @@ +# Directive: `python_generate_proto` + +This test case asserts that the `# gazelle:python_generate_proto` directive +correctly reads the name of the protobuf repository when bzlmod is being used. + +[gh-2994]: https://github.com/bazel-contrib/rules_python/issues/2994 diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/WORKSPACE b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/foo.proto b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/foo.proto new file mode 100644 index 0000000000..fe2af27aa6 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/test.yaml b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/test.yaml new file mode 100644 index 0000000000..36dd656b39 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf/test.yaml @@ -0,0 +1,3 @@ +--- +expect: + exit_code: 0 diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.in b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.in new file mode 100644 index 0000000000..4713404b19 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.in @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.out b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.out new file mode 100644 index 0000000000..686252f27c --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/BUILD.out @@ -0,0 +1,16 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/MODULE.bazel b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/MODULE.bazel new file mode 100644 index 0000000000..9ab4c175aa --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/MODULE.bazel @@ -0,0 +1 @@ +bazel_dep(name = "protobuf", version = "29.3", repo_name = "com_google_protobuf") diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/README.md b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/README.md new file mode 100644 index 0000000000..7900d49084 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/README.md @@ -0,0 +1,7 @@ +# Directive: `python_generate_proto` + +This test case asserts that the `# gazelle:python_generate_proto` directive +correctly reads the name of the protobuf repository when bzlmod is being used, +but the repository is renamed. + +[gh-2994]: https://github.com/bazel-contrib/rules_python/issues/2994 diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/WORKSPACE b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/foo.proto b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/foo.proto new file mode 100644 index 0000000000..fe2af27aa6 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/test.yaml b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/test.yaml new file mode 100644 index 0000000000..36dd656b39 --- /dev/null +++ b/gazelle/python/testdata/directive_python_generate_proto_bzlmod_protobuf_renamed/test.yaml @@ -0,0 +1,3 @@ +--- +expect: + exit_code: 0 diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index 8bf79cbc15..b76e1f92ec 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -98,6 +98,9 @@ const ( // separate pyi_deps attribute or merge type-checking dependencies into deps. // Defaults to false for backward compatibility. GeneratePyiDeps = "python_generate_pyi_deps" + // GenerateProto represents the directive that controls whether to generate + // python_generate_proto targets. + GenerateProto = "python_generate_proto" ) // GenerationModeType represents one of the generation modes for the Python @@ -186,6 +189,7 @@ type Config struct { labelNormalization LabelNormalizationType experimentalAllowRelativeImports bool generatePyiDeps bool + generateProto bool } type LabelNormalizationType int @@ -223,6 +227,7 @@ func New( labelNormalization: DefaultLabelNormalizationType, experimentalAllowRelativeImports: false, generatePyiDeps: false, + generateProto: false, } } @@ -257,6 +262,7 @@ func (c *Config) NewChild() *Config { labelNormalization: c.labelNormalization, experimentalAllowRelativeImports: c.experimentalAllowRelativeImports, generatePyiDeps: c.generatePyiDeps, + generateProto: c.generateProto, } } @@ -555,6 +561,16 @@ func (c *Config) GeneratePyiDeps() bool { return c.generatePyiDeps } +// SetGenerateProto sets whether py_proto_library should be generated for proto_library. +func (c *Config) SetGenerateProto(generateProto bool) { + c.generateProto = generateProto +} + +// GenerateProto returns whether py_proto_library should be generated for proto_library. +func (c *Config) GenerateProto() bool { + return c.generateProto +} + // FormatThirdPartyDependency returns a label to a third-party dependency performing all formating and normalization. func (c *Config) FormatThirdPartyDependency(repositoryName string, distributionName string) label.Label { conventionalDistributionName := strings.ReplaceAll(c.labelConvention, distributionNameLabelConventionSubstitution, distributionName) From a97b98ccbbab5070b088dd94efc485952ed8459e Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Mon, 14 Jul 2025 18:23:25 -0700 Subject: [PATCH 04/28] feat(gazelle): Add `include_pytest_conftest` annotation (#3080) Fixes #3076. Add a new gazelle annotation `include_pytest_conftest`. When unset or true, the gazelle behavior is unchanged. When false, gazelle will *not* inject the `:conftest` dependency to py_test targets. One of the refactorings that is done to support this is to pass around an `annotations` struct in `target.targetBuilder`. This will also open up support for other annotations in the future. --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 5 ++ gazelle/README.md | 85 +++++++++++++++++++ gazelle/python/generate.go | 17 +++- gazelle/python/parser.go | 30 ++++++- gazelle/python/target.go | 9 ++ .../README.md | 25 ++++++ .../WORKSPACE | 0 .../test.yaml | 5 ++ .../with_conftest/BUILD.in | 0 .../with_conftest/BUILD.out | 68 +++++++++++++++ .../with_conftest/bad_value_test.py | 1 + .../with_conftest/binary.py | 3 + .../with_conftest/conftest.py | 0 .../with_conftest/conftest_imported_test.py | 3 + .../with_conftest/conftest_included_test.py | 2 + .../with_conftest/false_test.py | 1 + .../with_conftest/falsey_test.py | 1 + .../with_conftest/last_value_wins_test.py | 6 ++ .../with_conftest/library.py | 1 + .../with_conftest/true_test.py | 1 + .../with_conftest/unset_test.py | 0 .../without_conftest/BUILD.in | 0 .../without_conftest/BUILD.out | 16 ++++ .../without_conftest/false_test.py | 1 + .../without_conftest/true_test.py | 1 + .../without_conftest/unset_test.py | 0 26 files changed, 275 insertions(+), 6 deletions(-) create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/README.md create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/WORKSPACE create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/test.yaml create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.in create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/bad_value_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/binary.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_imported_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_included_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/false_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/falsey_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/last_value_wins_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/library.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/true_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/unset_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/BUILD.in create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/BUILD.out create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/false_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/true_test.py create mode 100644 gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/unset_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e74f14b1db..b65a233f1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,11 @@ END_UNRELEASED_TEMPLATE * 3.12.11 * 3.13.5 * 3.14.0b3 +* (gazelle): New annotation `gazelle:include_pytest_conftest`. When not set (the + default) or `true`, gazelle will inject any `conftest.py` file found in the same + directory as a {obj}`py_test` target to that {obj}`py_test` target's `deps`. + This behavior is unchanged from previous versions. When `false`, the `:conftest` + dep is not added to the {obj}`py_test` target. * (gazelle) New directive `gazelle:python_generate_proto`; when `true`, Gazelle generates `py_proto_library` rules for `proto_library`. `false` by default. diff --git a/gazelle/README.md b/gazelle/README.md index 35a1e4f701..cf91461e39 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -550,6 +550,8 @@ The annotations are: | Tells Gazelle to ignore import statements. `imports` is a comma-separated list of imports to ignore. | | | [`# gazelle:include_dep targets`](#annotation-include_dep) | N/A | | Tells Gazelle to include a set of dependencies, even if they are not imported in a Python module. `targets` is a comma-separated list of target names to include as dependencies. | | +| [`# gazelle:include_pytest_conftest bool`](#annotation-include_pytest_conftest) | N/A | +| Whether or not to include a sibling `:conftest` target in the deps of a `py_test` target. Default behaviour is to include `:conftest`. | | #### Annotation: `ignore` @@ -622,6 +624,89 @@ deps = [ ] ``` +#### Annotation: `include_pytest_conftest` + +Added in [#3080][gh3080]. + +[gh3080]: https://github.com/bazel-contrib/rules_python/pull/3080 + +This annotation accepts any string that can be parsed by go's +[`strconv.ParseBool`][ParseBool]. If an unparsable string is passed, the +annotation is ignored. + +[ParseBool]: https://pkg.go.dev/strconv#ParseBool + +Starting with [`rules_python` 0.14.0][rules-python-0.14.0] (specifically [PR #879][gh879]), +Gazelle will include a `:conftest` dependency to an `py_test` target that is in +the same directory as `conftest.py`. + +[rules-python-0.14.0]: https://github.com/bazel-contrib/rules_python/releases/tag/0.14.0 +[gh879]: https://github.com/bazel-contrib/rules_python/pull/879 + +This annotation allows users to adjust that behavior. To disable the behavior, set +the annotation value to "false": + +``` +# some_file_test.py +# gazelle:include_pytest_conftest false +``` + +Example: + +Given a directory tree like: + +``` +. +├── BUILD.bazel +├── conftest.py +└── some_file_test.py +``` + +The default Gazelle behavior would create: + +```starlark +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "some_file_test", + srcs = ["some_file_test.py"], + deps = [":conftest"], +) +``` + +When `# gazelle:include_pytest_conftest false` is found in `some_file_test.py` + +```python +# some_file_test.py +# gazelle:include_pytest_conftest false +``` + +Gazelle will generate: + +```starlark +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "some_file_test", + srcs = ["some_file_test.py"], +) +``` + +See [Issue #3076][gh3076] for more information. + +[gh3076]: https://github.com/bazel-contrib/rules_python/issues/3076 + + #### Directive: `experimental_allow_relative_imports` Enables experimental support for resolving relative imports in `python_generation_mode package`. diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index 343743559f..279bee6af7 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -264,7 +264,9 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addSrc(filename). addModuleDependencies(mainModules[filename]). addResolvedDependencies(annotations.includeDeps). - generateImportsAttribute().build() + generateImportsAttribute(). + setAnnotations(*annotations). + build() result.Gen = append(result.Gen, pyBinary) result.Imports = append(result.Imports, pyBinary.PrivateAttr(config.GazelleImportsKey)) } @@ -305,6 +307,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addModuleDependencies(allDeps). addResolvedDependencies(annotations.includeDeps). generateImportsAttribute(). + setAnnotations(*annotations). build() if pyLibrary.IsEmpty(py.Kinds()[pyLibrary.Kind()]) { @@ -357,6 +360,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addSrc(pyBinaryEntrypointFilename). addModuleDependencies(deps). addResolvedDependencies(annotations.includeDeps). + setAnnotations(*annotations). generateImportsAttribute() pyBinary := pyBinaryTarget.build() @@ -387,6 +391,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addSrc(conftestFilename). addModuleDependencies(deps). addResolvedDependencies(annotations.includeDeps). + setAnnotations(*annotations). addVisibility(visibility). setTestonly(). generateImportsAttribute() @@ -418,6 +423,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes addSrcs(srcs). addModuleDependencies(deps). addResolvedDependencies(annotations.includeDeps). + setAnnotations(*annotations). generateImportsAttribute() } if (!cfg.PerPackageGenerationRequireTestEntryPoint() || hasPyTestEntryPointFile || hasPyTestEntryPointTarget || cfg.CoarseGrainedGeneration()) && !cfg.PerFileGeneration() { @@ -470,7 +476,14 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes for _, pyTestTarget := range pyTestTargets { if conftest != nil { - pyTestTarget.addModuleDependency(Module{Name: strings.TrimSuffix(conftestFilename, ".py")}) + conftestModule := Module{Name: strings.TrimSuffix(conftestFilename, ".py")} + if pyTestTarget.annotations.includePytestConftest == nil { + // unset; default behavior + pyTestTarget.addModuleDependency(conftestModule) + } else if *pyTestTarget.annotations.includePytestConftest { + // set; add if true, do not add if false + pyTestTarget.addModuleDependency(conftestModule) + } } pyTest := pyTestTarget.build() diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 11e01dbf51..3d0dbe7a5f 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -18,6 +18,8 @@ import ( "context" _ "embed" "fmt" + "log" + "strconv" "strings" "github.com/emirpasic/gods/sets/treeset" @@ -123,6 +125,7 @@ func (p *python3Parser) parse(pyFilenames *treeset.Set) (*treeset.Set, map[strin allAnnotations.ignore[k] = v } allAnnotations.includeDeps = append(allAnnotations.includeDeps, annotations.includeDeps...) + allAnnotations.includePytestConftest = annotations.includePytestConftest } allAnnotations.includeDeps = removeDupesFromStringTreeSetSlice(allAnnotations.includeDeps) @@ -183,8 +186,12 @@ const ( // The Gazelle annotation prefix. annotationPrefix string = "gazelle:" // The ignore annotation kind. E.g. '# gazelle:ignore '. - annotationKindIgnore annotationKind = "ignore" - annotationKindIncludeDep annotationKind = "include_dep" + annotationKindIgnore annotationKind = "ignore" + // Force a particular target to be added to `deps`. Multiple invocations are + // accumulated and the value can be comma separated. + // Eg: '# gazelle:include_dep //foo/bar:baz,@repo//:target + annotationKindIncludeDep annotationKind = "include_dep" + annotationKindIncludePytestConftest annotationKind = "include_pytest_conftest" ) // Comment represents a Python comment. @@ -222,6 +229,10 @@ type annotations struct { ignore map[string]struct{} // Labels that Gazelle should include as deps of the generated target. includeDeps []string + // Whether the conftest.py file, found in the same directory as the current + // python test file, should be added to the py_test target's `deps` attribute. + // A *bool is used so that we can handle the "not set" state. + includePytestConftest *bool } // annotationsFromComments returns all the annotations parsed out of the @@ -229,6 +240,7 @@ type annotations struct { func annotationsFromComments(comments []Comment) (*annotations, error) { ignore := make(map[string]struct{}) includeDeps := []string{} + var includePytestConftest *bool for _, comment := range comments { annotation, err := comment.asAnnotation() if err != nil { @@ -255,11 +267,21 @@ func annotationsFromComments(comments []Comment) (*annotations, error) { includeDeps = append(includeDeps, t) } } + if annotation.kind == annotationKindIncludePytestConftest { + val := annotation.value + parsedVal, err := strconv.ParseBool(val) + if err != nil { + log.Printf("WARNING: unable to cast %q to bool in %q. Ignoring annotation", val, comment) + continue + } + includePytestConftest = &parsedVal + } } } return &annotations{ - ignore: ignore, - includeDeps: includeDeps, + ignore: ignore, + includeDeps: includeDeps, + includePytestConftest: includePytestConftest, }, nil } diff --git a/gazelle/python/target.go b/gazelle/python/target.go index 06b653d915..6e6c3f4b14 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -37,6 +37,7 @@ type targetBuilder struct { main *string imports []string testonly bool + annotations *annotations } // newTargetBuilder constructs a new targetBuilder. @@ -51,6 +52,7 @@ func newTargetBuilder(kind, name, pythonProjectRoot, bzlPackage string, siblingS deps: treeset.NewWith(moduleComparator), resolvedDeps: treeset.NewWith(godsutils.StringComparator), visibility: treeset.NewWith(godsutils.StringComparator), + annotations: new(annotations), } } @@ -130,6 +132,13 @@ func (t *targetBuilder) setTestonly() *targetBuilder { return t } +// setAnnotations sets the annotations attribute on the target. +func (t *targetBuilder) setAnnotations(val annotations) *targetBuilder { + t.annotations = &val + return t +} + + // generateImportsAttribute generates the imports attribute. // These are a list of import directories to be added to the PYTHONPATH. In our // case, the value we add is on Bazel sub-packages to be able to perform imports diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/README.md b/gazelle/python/testdata/annotation_include_pytest_conftest/README.md new file mode 100644 index 0000000000..6a347d154e --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/README.md @@ -0,0 +1,25 @@ +# Annotation: Include Pytest Conftest + +Validate that the `# gazelle:include_pytest_conftest` annotation follows +this logic: + ++ When a `conftest.py` file does not exist: + + all values have no affect ++ When a `conftest.py` file does exist: + + Truthy values add `:conftest` to `deps`. + + Falsey values do not add `:conftest` to `deps`. + + Unset (no annotation) performs the default action. + +Additionally, we test that: + ++ invalid values (eg `foo`) print a warning and then act as if + the annotation was not present. ++ last annotation (highest line number) wins. ++ the annotation has no effect on non-test files/targets. ++ the `include_dep` can still inject `:conftest` even when `include_pytest_conftest` + is false. ++ `import conftest` will still add the dep even when `include_pytest_conftest` is + false. + +An annotation without a value is not tested, as that's part of the core +annotation framework and not specific to this annotation. diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/WORKSPACE b/gazelle/python/testdata/annotation_include_pytest_conftest/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/test.yaml b/gazelle/python/testdata/annotation_include_pytest_conftest/test.yaml new file mode 100644 index 0000000000..e643d0e90c --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/test.yaml @@ -0,0 +1,5 @@ +--- +expect: + stderr: | + gazelle: WARNING: unable to cast "foo" to bool in "# gazelle:include_pytest_conftest foo". Ignoring annotation + exit_code: 0 diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.in b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out new file mode 100644 index 0000000000..60695352ca --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out @@ -0,0 +1,68 @@ +load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") + +py_binary( + name = "binary", + srcs = ["binary.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "with_conftest", + srcs = [ + "binary.py", + "library.py", + ], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "bad_value_test", + srcs = ["bad_value_test.py"], + deps = [":conftest"], +) + +py_test( + name = "conftest_imported_test", + srcs = ["conftest_imported_test.py"], + deps = [":conftest"], +) + +py_test( + name = "conftest_included_test", + srcs = ["conftest_included_test.py"], + deps = [":conftest"], +) + +py_test( + name = "false_test", + srcs = ["false_test.py"], +) + +py_test( + name = "falsey_test", + srcs = ["falsey_test.py"], +) + +py_test( + name = "last_value_wins_test", + srcs = ["last_value_wins_test.py"], +) + +py_test( + name = "true_test", + srcs = ["true_test.py"], + deps = [":conftest"], +) + +py_test( + name = "unset_test", + srcs = ["unset_test.py"], + deps = [":conftest"], +) diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/bad_value_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/bad_value_test.py new file mode 100644 index 0000000000..af2e8c54e0 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/bad_value_test.py @@ -0,0 +1 @@ +# gazelle:include_pytest_conftest foo diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/binary.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/binary.py new file mode 100644 index 0000000000..d6dc8413d4 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/binary.py @@ -0,0 +1,3 @@ +# gazelle:include_pytest_conftest true +if __name__ == "__main__": + pass diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_imported_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_imported_test.py new file mode 100644 index 0000000000..2c72ca4df1 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_imported_test.py @@ -0,0 +1,3 @@ +import conftest + +# gazelle:include_pytest_conftest false diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_included_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_included_test.py new file mode 100644 index 0000000000..c942bfb1ab --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/conftest_included_test.py @@ -0,0 +1,2 @@ +# gazelle:include_dep :conftest +# gazelle:include_pytest_conftest false diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/false_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/false_test.py new file mode 100644 index 0000000000..ba71a2818b --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/false_test.py @@ -0,0 +1 @@ +# gazelle:include_pytest_conftest false diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/falsey_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/falsey_test.py new file mode 100644 index 0000000000..c4387b3a8c --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/falsey_test.py @@ -0,0 +1 @@ +# gazelle:include_pytest_conftest 0 diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/last_value_wins_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/last_value_wins_test.py new file mode 100644 index 0000000000..6ffc06f9c0 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/last_value_wins_test.py @@ -0,0 +1,6 @@ +# gazelle:include_pytest_conftest true +# gazelle:include_pytest_conftest TRUE +# gazelle:include_pytest_conftest False +# gazelle:include_pytest_conftest 0 +# gazelle:include_pytest_conftest 1 +# gazelle:include_pytest_conftest F diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/library.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/library.py new file mode 100644 index 0000000000..b2d10359da --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/library.py @@ -0,0 +1 @@ +# gazelle:include_pytest_conftest true diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/true_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/true_test.py new file mode 100644 index 0000000000..b2d10359da --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/true_test.py @@ -0,0 +1 @@ +# gazelle:include_pytest_conftest true diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/unset_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/unset_test.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/BUILD.in b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/BUILD.out b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/BUILD.out new file mode 100644 index 0000000000..01383344c5 --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/BUILD.out @@ -0,0 +1,16 @@ +load("@rules_python//python:defs.bzl", "py_test") + +py_test( + name = "false_test", + srcs = ["false_test.py"], +) + +py_test( + name = "true_test", + srcs = ["true_test.py"], +) + +py_test( + name = "unset_test", + srcs = ["unset_test.py"], +) diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/false_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/false_test.py new file mode 100644 index 0000000000..ba71a2818b --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/false_test.py @@ -0,0 +1 @@ +# gazelle:include_pytest_conftest false diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/true_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/true_test.py new file mode 100644 index 0000000000..b2d10359da --- /dev/null +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/true_test.py @@ -0,0 +1 @@ +# gazelle:include_pytest_conftest true diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/unset_test.py b/gazelle/python/testdata/annotation_include_pytest_conftest/without_conftest/unset_test.py new file mode 100644 index 0000000000..e69de29bb2 From 65a1c8541b8fdf45700bae110f2ae7d7a311c9cd Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Mon, 14 Jul 2025 18:29:31 -0700 Subject: [PATCH 05/28] docs: tell how to emulate dependency groups with pip-compile (#3089) While pyproject.toml is supported by piptools, dependency groups aren't. They can be emulated by using multiple files. Explain how to do that in the docs and link to the upstream feature request (https://github.com/jazzband/pip-tools/issues/2062) Along the way, link to our own feature request for pylock.toml support in pip.parse. --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- docs/pypi/lock.md | 28 ++++++++++++++++++++++++++-- python/private/pypi/pip_compile.bzl | 7 +++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/pypi/lock.md b/docs/pypi/lock.md index db557fe594..b5d8ec24f7 100644 --- a/docs/pypi/lock.md +++ b/docs/pypi/lock.md @@ -5,6 +5,8 @@ :::{note} Currently `rules_python` only supports `requirements.txt` format. + +#{gh-issue}`2787` tracks `pylock.toml` support. ::: ## requirements.txt @@ -37,11 +39,33 @@ This rule generates two targets: Once you generate this fully specified list of requirements, you can install the requirements ([bzlmod](./download)/[WORKSPACE](./download-workspace)). :::{warning} -If you're specifying dependencies in `pyproject.toml`, make sure to include the `[build-system]` configuration, with pinned dependencies. `compile_pip_requirements` will use the build system specified to read your project's metadata, and you might see non-hermetic behavior if you don't pin the build system. +If you're specifying dependencies in `pyproject.toml`, make sure to include the +`[build-system]` configuration, with pinned dependencies. +`compile_pip_requirements` will use the build system specified to read your +project's metadata, and you might see non-hermetic behavior if you don't pin the +build system. -Not specifying `[build-system]` at all will result in using a default `[build-system]` configuration, which uses unpinned versions ([ref](https://peps.python.org/pep-0518/#build-system-table)). +Not specifying `[build-system]` at all will result in using a default +`[build-system]` configuration, which uses unpinned versions +([ref](https://peps.python.org/pep-0518/#build-system-table)). ::: + +#### pip compile Dependency groups + +pip-compile doesn't yet support pyproject.toml dependency groups. Follow +[pip-tools #2062](https://github.com/jazzband/pip-tools/issues/2062) +to see the status of their support. + +In the meantime, support can be emulated by passing multiple files to `srcs`: + +```starlark +compile_pip_requirements( + srcs = ["pyproject.toml", "requirements-dev.in"] + ... +) +``` + ### uv pip compile (bzlmod only) We also have experimental setup for the `uv pip compile` way of generating lock files. diff --git a/python/private/pypi/pip_compile.bzl b/python/private/pypi/pip_compile.bzl index 2e3e530153..28923005df 100644 --- a/python/private/pypi/pip_compile.bzl +++ b/python/private/pypi/pip_compile.bzl @@ -40,7 +40,7 @@ def pip_compile( tags = None, constraints = [], **kwargs): - """Generates targets for managing pip dependencies with pip-compile. + """Generates targets for managing pip dependencies with pip-compile (piptools). By default this rules generates a filegroup named "[name]" which can be included in the data of some other compile_pip_requirements rule that references these requirements @@ -65,7 +65,10 @@ def pip_compile( * a requirements text file, usually named `requirements.in` * A `.toml` file, where the `project.dependencies` list is used as per [PEP621](https://peps.python.org/pep-0621/). - extra_args: passed to pip-compile. + extra_args: passed to pip-compile (aka `piptools`). See the + [pip-compile docs](https://pip-tools.readthedocs.io/en/latest/cli/pip-compile) + for args and meaning (passing `-h` and/or `--version` can help + inform what args are available) extra_deps: extra dependencies passed to pip-compile. generate_hashes: whether to put hashes in the requirements_txt file. py_binary: the py_binary rule to be used. From 9555ba8e9ce0902f3d2a315a6e19b8bebe79c0ce Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 15 Jul 2025 11:49:48 +0900 Subject: [PATCH 06/28] chore: update python toolchains (#3074) - use the SHA256SUMS file instead of individual sha256sum files. This improves the speed of the tooling and also the old files just disappeared for the latest toolchain release. - update to the latest release. --- CHANGELOG.md | 6 +- python/private/print_toolchain_checksums.bzl | 41 +++-- python/versions.bzl | 168 +++++++++---------- tests/python/python_tests.bzl | 2 +- 4 files changed, 107 insertions(+), 110 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b65a233f1e..c7a5a8fad5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,12 +60,12 @@ END_UNRELEASED_TEMPLATE * (gazelle) Types for exposed members of `python.ParserOutput` are now all public. * (gazelle) Removed the requirement for `__init__.py`, `__main__.py`, or `__test__.py` files to be present in a directory to generate a `BUILD.bazel` file. -* (toolchain) Updated the following toolchains to build 20250702 to patch CVE-2025-47273: +* (toolchain) Updated the following toolchains to build 20250708 to patch CVE-2025-47273: * 3.9.23 * 3.10.18 * 3.11.13 * 3.12.11 - * 3.14.0b3 + * 3.14.0b4 * (toolchain) Python 3.13 now references 3.13.5 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) @@ -105,7 +105,7 @@ END_UNRELEASED_TEMPLATE * 3.11.13 * 3.12.11 * 3.13.5 - * 3.14.0b3 + * 3.14.0b4 * (gazelle): New annotation `gazelle:include_pytest_conftest`. When not set (the default) or `true`, gazelle will inject any `conftest.py` file found in the same directory as a {obj}`py_test` target to that {obj}`py_test` target's `deps`. diff --git a/python/private/print_toolchain_checksums.bzl b/python/private/print_toolchain_checksums.bzl index eaaa5b9d75..bd370baf10 100644 --- a/python/private/print_toolchain_checksums.bzl +++ b/python/private/print_toolchain_checksums.bzl @@ -28,6 +28,7 @@ def print_toolchains_checksums(name): template = """\ cat > "$@" <<'EOF' #!/bin/bash +set -euo pipefail set -o errexit -o nounset -o pipefail @@ -54,28 +55,9 @@ EOF def _commands_for_version(*, python_version, metadata): lines = [] - lines += [ - "cat < Date: Tue, 15 Jul 2025 14:24:31 +0200 Subject: [PATCH 07/28] feat: replace /bin/bash with /usr/bin/env bash (#3087) This allows system which don't have this location (looking at you NixOS) to run the scripts. --- CHANGELOG.md | 1 + addlicense.sh | 2 +- docs/readthedocs_build.sh | 2 +- python/private/interpreter_tmpl.sh | 2 +- python/private/print_toolchain_checksums.bzl | 2 +- python/private/stage1_bootstrap_template.sh | 2 +- python/uv/private/lock.sh | 2 +- sphinxdocs/private/sphinx_run_template.sh | 2 +- tests/bootstrap_impls/external_binary_test.sh | 2 +- tests/integration/bazel_from_env | 2 +- 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7a5a8fad5..7248e1f9f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,7 @@ END_UNRELEASED_TEMPLATE * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) * (ci) We are now testing on Ubuntu 22.04 for RBE and non-RBE configurations. +* (core) #!/usr/bin/env bash is now used as a shebang in the stage1 bootstrap template. {#v0-0-0-fixed} ### Fixed diff --git a/addlicense.sh b/addlicense.sh index 8cc8fb33bc..8dc82bbcc9 100755 --- a/addlicense.sh +++ b/addlicense.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Copyright 2023 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/docs/readthedocs_build.sh b/docs/readthedocs_build.sh index 3f67310197..ec5390bfc7 100755 --- a/docs/readthedocs_build.sh +++ b/docs/readthedocs_build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -eou pipefail diff --git a/python/private/interpreter_tmpl.sh b/python/private/interpreter_tmpl.sh index cfe85ec1be..c4e87fbb43 100644 --- a/python/private/interpreter_tmpl.sh +++ b/python/private/interpreter_tmpl.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # --- begin runfiles.bash initialization v3 --- # Copy-pasted from the Bazel Bash runfiles library v3. diff --git a/python/private/print_toolchain_checksums.bzl b/python/private/print_toolchain_checksums.bzl index bd370baf10..b4fa400221 100644 --- a/python/private/print_toolchain_checksums.bzl +++ b/python/private/print_toolchain_checksums.bzl @@ -27,7 +27,7 @@ def print_toolchains_checksums(name): template = """\ cat > "$@" <<'EOF' -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail set -o errexit -o nounset -o pipefail diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh index d992b55cae..9927d4faa7 100644 --- a/python/private/stage1_bootstrap_template.sh +++ b/python/private/stage1_bootstrap_template.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -e diff --git a/python/uv/private/lock.sh b/python/uv/private/lock.sh index b6ba0c6c48..ffb19b2bea 100755 --- a/python/uv/private/lock.sh +++ b/python/uv/private/lock.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail if [[ -n "${BUILD_WORKSPACE_DIRECTORY:-}" ]]; then diff --git a/sphinxdocs/private/sphinx_run_template.sh b/sphinxdocs/private/sphinx_run_template.sh index 4a1f1e4410..aa83757c1b 100644 --- a/sphinxdocs/private/sphinx_run_template.sh +++ b/sphinxdocs/private/sphinx_run_template.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash declare -a args %SETUP_ARGS% diff --git a/tests/bootstrap_impls/external_binary_test.sh b/tests/bootstrap_impls/external_binary_test.sh index e3516af18e..92799354d6 100755 --- a/tests/bootstrap_impls/external_binary_test.sh +++ b/tests/bootstrap_impls/external_binary_test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euxo pipefail tmpdir="${TEST_TMPDIR}/external_binary" diff --git a/tests/integration/bazel_from_env b/tests/integration/bazel_from_env index 96780b8156..a372736f32 100755 --- a/tests/integration/bazel_from_env +++ b/tests/integration/bazel_from_env @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # A simple wrapper so rules_bazel_integration_test can use the # bazel version inherited from the environment. From 6fa4459a877aa1df1dc26320182c403d00e003c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 21:24:58 +0900 Subject: [PATCH 08/28] build(deps): bump certifi from 2025.6.15 to 2025.7.14 in /docs (#3092) [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=certifi&package-manager=pip&previous-version=2025.6.15&new-version=2025.7.14)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 7a32ff7716..9e46724770 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -17,9 +17,9 @@ babel==2.17.0 \ --hash=sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d \ --hash=sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2 # via sphinx -certifi==2025.6.15 \ - --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ - --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 # via requests charset-normalizer==3.4.2 \ --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ From cab415d82ebbfd1b8b2d81ef61baf3f9a274a1b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 21:25:26 +0900 Subject: [PATCH 09/28] build(deps): bump certifi from 2025.6.15 to 2025.7.14 in /tools/publish (#3095) [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=certifi&package-manager=pip&previous-version=2025.6.15&new-version=2025.7.14)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tools/publish/requirements_darwin.txt | 6 +++--- tools/publish/requirements_linux.txt | 6 +++--- tools/publish/requirements_universal.txt | 6 +++--- tools/publish/requirements_windows.txt | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/publish/requirements_darwin.txt b/tools/publish/requirements_darwin.txt index dab86f3adc..677cc6f7eb 100644 --- a/tools/publish/requirements_darwin.txt +++ b/tools/publish/requirements_darwin.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.6.15 \ - --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ - --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 # via requests charset-normalizer==3.4.2 \ --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ diff --git a/tools/publish/requirements_linux.txt b/tools/publish/requirements_linux.txt index c9d25eab58..98f119b3c9 100644 --- a/tools/publish/requirements_linux.txt +++ b/tools/publish/requirements_linux.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.6.15 \ - --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ - --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 # via requests cffi==1.17.1 \ --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ diff --git a/tools/publish/requirements_universal.txt b/tools/publish/requirements_universal.txt index a642e9280d..58625a4aad 100644 --- a/tools/publish/requirements_universal.txt +++ b/tools/publish/requirements_universal.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 ; python_full_version < '3.12' \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.6.15 \ - --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ - --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 # via requests cffi==1.17.1 ; platform_python_implementation != 'PyPy' and sys_platform == 'linux' \ --hash=sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8 \ diff --git a/tools/publish/requirements_windows.txt b/tools/publish/requirements_windows.txt index d3944056c0..374541d96f 100644 --- a/tools/publish/requirements_windows.txt +++ b/tools/publish/requirements_windows.txt @@ -6,9 +6,9 @@ backports-tarfile==1.2.0 \ --hash=sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34 \ --hash=sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991 # via jaraco-context -certifi==2025.6.15 \ - --hash=sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057 \ - --hash=sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b +certifi==2025.7.14 \ + --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \ + --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995 # via requests charset-normalizer==3.4.2 \ --hash=sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4 \ From 2c7187d6558ce39e1bd2fea4c0637722258f7790 Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Tue, 15 Jul 2025 22:34:01 -0700 Subject: [PATCH 10/28] fix: support debian multiarch with local toolchains (#3100) Apparently, there is a "multiarch" style of Python installations, where the shared libraries can be in a sub-directory. The best docs I could find about this are https://wiki.debian.org/Python/MultiArch. To fix, looking at the `MULTIARCH` sysconfig var tells what subdirectory, if any should be looked in. Along the way, fix a changelog issue reference url. Fixes https://github.com/bazel-contrib/rules_python/issues/3099 --- CHANGELOG.md | 4 +++- python/private/get_local_runtime_info.py | 8 ++++++++ python/private/local_runtime_repo.bzl | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7248e1f9f2..b36ceafa27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,10 +85,12 @@ END_UNRELEASED_TEMPLATE ([#2503](https://github.com/bazel-contrib/rules_python/issues/2503)). * (toolchains) `local_runtime_repo` now checks if the include directory exists before attempting to watch it, fixing issues on macOS with system Python - ({gh-issue}`3043`). + ([#3043](https://github.com/bazel-contrib/rules_python/issues/3043)). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle. +* (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8 + ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)). {#v0-0-0-added} ### Added diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py index 19db3a2935..c8371357c2 100644 --- a/python/private/get_local_runtime_info.py +++ b/python/private/get_local_runtime_info.py @@ -35,7 +35,15 @@ # of settings. # https://stackoverflow.com/questions/47423246/get-pythons-lib-path # For now, it seems LIBDIR has what is needed, so just use that. + # See also: MULTIARCH "LIBDIR", + # On Debian, with multiarch enabled, prior to Python 3.10, `LIBDIR` didn't + # tell the location of the libs, just the base directory. The `MULTIARCH` + # sysconfig variable tells the subdirectory within it with the libs. + # See: + # https://wiki.debian.org/Python/MultiArch + # https://git.launchpad.net/ubuntu/+source/python3.12/tree/debian/changelog#n842 + "MULTIARCH", # The versioned libpythonX.Y.so.N file. Usually? # It might be a static archive (.a) file instead. "INSTSONAME", diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl index 3b4b4c020d..b8b7164b54 100644 --- a/python/private/local_runtime_repo.bzl +++ b/python/private/local_runtime_repo.bzl @@ -126,6 +126,7 @@ def _local_runtime_repo_impl(rctx): # In some cases, the same value is returned for multiple keys. Not clear why. shared_lib_names = {v: None for v in shared_lib_names}.keys() shared_lib_dir = info["LIBDIR"] + multiarch = info["MULTIARCH"] # The specific files are symlinked instead of the whole directory # because it can point to a directory that has more than just @@ -135,6 +136,11 @@ def _local_runtime_repo_impl(rctx): for name in shared_lib_names: origin = rctx.path("{}/{}".format(shared_lib_dir, name)) + # If the origin doesn't exist, try the multiarch location, in case + # it's an older Python / Debian release. + if not origin.exists and multiarch: + origin = rctx.path("{}/{}/{}".format(shared_lib_dir, multiarch, name)) + # The reported names don't always exist; it depends on the particulars # of the runtime installation. if origin.exists: From f02c9c72f5f0d63ce10da7b7393f58bc541e835a Mon Sep 17 00:00:00 2001 From: Alex Eagle Date: Thu, 17 Jul 2025 12:34:52 -0700 Subject: [PATCH 11/28] refactor(gazelle_manifest): print the wrong hash when encountered (#3103) Ideally developers should always re-run the manifest generator, which will update the hash for them. However I've got clients where the CI system is the only place that all the dependencies can resolve, either because of credentials needed to access the wheelhouse/PyPI, or because of disk exhaustion from massive wheels. Printing the difference allows a red PR to be greened up just by reading the CI log. --- gazelle/manifest/manifest.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gazelle/manifest/manifest.go b/gazelle/manifest/manifest.go index 26b0dfb394..c5cd8a7d69 100644 --- a/gazelle/manifest/manifest.go +++ b/gazelle/manifest/manifest.go @@ -70,6 +70,9 @@ func (f *File) VerifyIntegrity(manifestGeneratorHashFile, requirements io.Reader return false, fmt.Errorf("failed to verify integrity: %w", err) } valid := (f.Integrity == fmt.Sprintf("%x", integrityBytes)) + if (!valid) { + fmt.Printf("WARN: Integrity hash was %v but expected %x\n", f.Integrity, integrityBytes) + } return valid, nil } From 004be45d2bf9924fc6805445d34885b0e1d6283d Mon Sep 17 00:00:00 2001 From: Charles OuGuo Date: Sun, 20 Jul 2025 16:54:06 -0400 Subject: [PATCH 12/28] feat(gazelle): `python_proto_naming_convention` directive controls `py_proto_library` naming (#3093) Closes https://github.com/bazel-contrib/rules_python/issues/3081. This adds support in the Gazelle plugin for controlling how the generated `py_proto_library` rules are named; support for these was originally added in https://github.com/bazel-contrib/rules_python/pull/3057. We do this via a new Gazelle directive, `python_proto_naming_convention`, which is similar to `python_library_naming_convention` and the like, except it interpolates `$proto_name$`, which is the `proto_library` rule minus any trailing `_proto`. We default to `$proto_name$_py_pb2`. For instance, for a `proto_library` named `foo_proto`, the default value would generate `foo_py_pb2`, aligning with [the convention stated in the Bazel docs.](https://bazel.build/reference/be/protocol-buffer#py_proto_library) --- CHANGELOG.md | 2 ++ gazelle/README.md | 27 +++++++++++++++++++ gazelle/python/configure.go | 3 +++ gazelle/python/generate.go | 17 +++++++++--- .../README.md | 9 +++++++ .../WORKSPACE | 1 + .../test.yaml | 3 +++ .../BUILD.in | 17 ++++++++++++ .../BUILD.out | 17 ++++++++++++ .../foo.proto | 7 +++++ .../BUILD.in | 9 +++++++ .../BUILD.out | 16 +++++++++++ .../foo.proto | 7 +++++ .../BUILD.in | 10 +++++++ .../BUILD.out | 17 ++++++++++++ .../foo.proto | 7 +++++ .../BUILD.in | 16 +++++++++++ .../BUILD.out | 17 ++++++++++++ .../foo.proto | 7 +++++ gazelle/pythonconfig/pythonconfig.go | 21 +++++++++++++++ 20 files changed, 227 insertions(+), 3 deletions(-) create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/README.md create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/WORKSPACE create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test.yaml create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/foo.proto create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/foo.proto create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/foo.proto create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.in create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.out create mode 100644 gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/foo.proto diff --git a/CHANGELOG.md b/CHANGELOG.md index b36ceafa27..d54d3a8881 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,8 @@ END_UNRELEASED_TEMPLATE dep is not added to the {obj}`py_test` target. * (gazelle) New directive `gazelle:python_generate_proto`; when `true`, Gazelle generates `py_proto_library` rules for `proto_library`. `false` by default. +* (gazelle) New directive `gazelle:python_proto_naming_convention`; controls + naming of `py_proto_library` rules. {#v0-0-0-removed} ### Removed diff --git a/gazelle/README.md b/gazelle/README.md index cf91461e39..222c1171ab 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -208,6 +208,8 @@ Python-specific directives are as follows: | Controls the `py_binary` naming convention. Follows the same interpolation rules as `python_library_naming_convention`. | | | `# gazelle:python_test_naming_convention` | `$package_name$_test` | | Controls the `py_test` naming convention. Follows the same interpolation rules as `python_library_naming_convention`. | | +| [`# gazelle:python_proto_naming_convention`](#directive-python_proto_naming_convention) | `$proto_name$_py_pb2` | +| Controls the `py_proto_library` naming convention. It interpolates `$proto_name$` with the proto_library rule name, minus any trailing _proto. E.g. if the proto_library name is `foo_proto`, setting this to `$proto_name$_my_lib` would render to `foo_my_lib`. | | | `# gazelle:resolve py ...` | n/a | | Instructs the plugin what target to add as a dependency to satisfy a given import statement. The syntax is `# gazelle:resolve py import-string label` where `import-string` is the symbol in the python `import` statement, and `label` is the Bazel label that Gazelle should write in `deps`. | | | [`# gazelle:python_default_visibility labels`](#directive-python_default_visibility) | | @@ -262,6 +264,31 @@ py_libary( [python-packaging-user-guide]: https://github.com/pypa/packaging.python.org/blob/4c86169a/source/tutorials/packaging-projects.rst +#### Directive: `python_proto_naming_convention`: + +Set this directive to a string pattern to control how the generated `py_proto_library` targets are named. When generating new `py_proto_library` rules, Gazelle will replace `$proto_name$` in the pattern with the name of the `proto_library` rule, stripping out a trailing `_proto`. For example: + +```starlark +# gazelle:python_generate_proto true +# gazelle:python_proto_naming_convention my_custom_$proto_name$_pattern + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) +``` + +produces the following `py_proto_library` rule: +```starlark +py_proto_library( + name = "my_custom_foo_pattern", + deps = [":foo_proto"], +) +``` + +The default naming convention is `$proto_name$_pb2_py`, so by default in the above example Gazelle would generate `foo_pb2_py`. Any pre-existing rules are left in place and not renamed. + +Note that the Python library will always be imported as `foo_pb2` in Python code, regardless of the naming convention. Also note that Gazelle is currently not able to map said imports, e.g. `import foo_pb2`, to fill in `py_proto_library` targets as dependencies of other rules. See [this issue](https://github.com/bazel-contrib/rules_python/issues/1703). #### Directive: `python_default_visibility`: diff --git a/gazelle/python/configure.go b/gazelle/python/configure.go index 7131be283d..079f1d84d4 100644 --- a/gazelle/python/configure.go +++ b/gazelle/python/configure.go @@ -63,6 +63,7 @@ func (py *Configurer) KnownDirectives() []string { pythonconfig.LibraryNamingConvention, pythonconfig.BinaryNamingConvention, pythonconfig.TestNamingConvention, + pythonconfig.ProtoNamingConvention, pythonconfig.DefaultVisibilty, pythonconfig.Visibility, pythonconfig.TestFilePattern, @@ -179,6 +180,8 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) { config.SetBinaryNamingConvention(strings.TrimSpace(d.Value)) case pythonconfig.TestNamingConvention: config.SetTestNamingConvention(strings.TrimSpace(d.Value)) + case pythonconfig.ProtoNamingConvention: + config.SetProtoNamingConvention(strings.TrimSpace(d.Value)) case pythonconfig.DefaultVisibilty: switch directiveArg := strings.TrimSpace(d.Value); directiveArg { case "NONE": diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index 279bee6af7..5b6ba79d69 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -227,7 +227,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes result.Gen = make([]*rule.Rule, 0) if cfg.GenerateProto() { - generateProtoLibraries(args, pythonProjectRoot, visibility, &result) + generateProtoLibraries(args, cfg, pythonProjectRoot, visibility, &result) } collisionErrors := singlylinkedlist.New() @@ -569,7 +569,7 @@ func ensureNoCollision(file *rule.File, targetName, kind string) error { return nil } -func generateProtoLibraries(args language.GenerateArgs, pythonProjectRoot string, visibility []string, res *language.GenerateResult) { +func generateProtoLibraries(args language.GenerateArgs, cfg *pythonconfig.Config, pythonProjectRoot string, visibility []string, res *language.GenerateResult) { // First, enumerate all the proto_library in this package. var protoRuleNames []string for _, r := range args.OtherGen { @@ -582,10 +582,16 @@ func generateProtoLibraries(args language.GenerateArgs, pythonProjectRoot string // Next, enumerate all the pre-existing py_proto_library in this package, so we can delete unnecessary rules later. pyProtoRules := map[string]bool{} + pyProtoRulesForProto := map[string]string{} if args.File != nil { for _, r := range args.File.Rules { if r.Kind() == "py_proto_library" { pyProtoRules[r.Name()] = false + + protos := r.AttrStrings("deps") + for _, proto := range protos { + pyProtoRulesForProto[strings.TrimPrefix(proto, ":")] = r.Name() + } } } } @@ -593,7 +599,12 @@ func generateProtoLibraries(args language.GenerateArgs, pythonProjectRoot string emptySiblings := treeset.Set{} // Generate a py_proto_library for each proto_library. for _, protoRuleName := range protoRuleNames { - pyProtoLibraryName := strings.TrimSuffix(protoRuleName, "_proto") + "_py_pb2" + pyProtoLibraryName := cfg.RenderProtoName(protoRuleName) + if ruleName, ok := pyProtoRulesForProto[protoRuleName]; ok { + // There exists a pre-existing py_proto_library for this proto. Keep this name. + pyProtoLibraryName = ruleName + } + pyProtoLibrary := newTargetBuilder(pyProtoLibraryKind, pyProtoLibraryName, pythonProjectRoot, args.Rel, &emptySiblings). addVisibility(visibility). addResolvedDependency(":" + protoRuleName). diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/README.md b/gazelle/python/testdata/directive_python_proto_naming_convention/README.md new file mode 100644 index 0000000000..594379cdfc --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/README.md @@ -0,0 +1,9 @@ +# Directive: `python_proto_naming_convention` + +This test case asserts that the `# gazelle:python_proto_naming_convention` directive +correctly: + +1. Has no effect on pre-existing `py_proto_library` when `gazelle:python_generate_proto` is disabled. +2. Uses the default value when proto generation is on and `python_proto_naming_convention` is not set. +3. Uses the provided naming convention when proto generation is on and `python_proto_naming_convention` is set. +4. With a pre-existing `py_proto_library` not following a given naming convention, keeps it intact and does not rename it. \ No newline at end of file diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/WORKSPACE b/gazelle/python/testdata/directive_python_proto_naming_convention/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test.yaml b/gazelle/python/testdata/directive_python_proto_naming_convention/test.yaml new file mode 100644 index 0000000000..36dd656b39 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test.yaml @@ -0,0 +1,3 @@ +--- +expect: + exit_code: 0 diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.in b/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.in new file mode 100644 index 0000000000..2171d877f4 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.in @@ -0,0 +1,17 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto false +# gazelle:python_proto_naming_convention some_$proto_name$_value + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_proto_custom_name", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.out b/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.out new file mode 100644 index 0000000000..2171d877f4 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/BUILD.out @@ -0,0 +1,17 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto false +# gazelle:python_proto_naming_convention some_$proto_name$_value + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_proto_custom_name", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/foo.proto b/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/foo.proto new file mode 100644 index 0000000000..022e29ae69 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test1_python_generation_disabled_does_nothing/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo.bar; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.in b/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.in new file mode 100644 index 0000000000..4713404b19 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.in @@ -0,0 +1,9 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.out b/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.out new file mode 100644 index 0000000000..686252f27c --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/BUILD.out @@ -0,0 +1,16 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/foo.proto b/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/foo.proto new file mode 100644 index 0000000000..fe2af27aa6 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test2_python_generation_enabled_uses_default/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.in b/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.in new file mode 100644 index 0000000000..b68a9937dc --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.in @@ -0,0 +1,10 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true +# gazelle:python_proto_naming_convention some_$proto_name$_value + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.out b/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.out new file mode 100644 index 0000000000..f432e9a0c3 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/BUILD.out @@ -0,0 +1,17 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true +# gazelle:python_proto_naming_convention some_$proto_name$_value + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "some_foo_value", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/foo.proto b/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/foo.proto new file mode 100644 index 0000000000..fe2af27aa6 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test3_python_generation_enabled_uses_value/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Foo { + string bar = 1; +} diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.in b/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.in new file mode 100644 index 0000000000..cc7d120a7e --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.in @@ -0,0 +1,16 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true +# gazelle:python_proto_naming_convention $proto_name$_bar + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_proto", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.out b/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.out new file mode 100644 index 0000000000..080b83f1fb --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/BUILD.out @@ -0,0 +1,17 @@ +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true +# gazelle:python_proto_naming_convention $proto_name$_bar + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_proto", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) diff --git a/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/foo.proto b/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/foo.proto new file mode 100644 index 0000000000..fe2af27aa6 --- /dev/null +++ b/gazelle/python/testdata/directive_python_proto_naming_convention/test4_python_generation_enabled_with_preexisting_keeps_intact/foo.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package foo; + +message Foo { + string bar = 1; +} diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index b76e1f92ec..001fd334a4 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -74,6 +74,12 @@ const ( // naming convention. See python_library_naming_convention for more info on // the package name interpolation. TestNamingConvention = "python_test_naming_convention" + // ProtoNamingConvention represents the directive that controls the + // py_proto_library naming convention. It interpolates $proto_name$ with + // the proto_library rule name, minus any trailing _proto. E.g. if the + // proto_library name is `foo_proto`, setting this to `$proto_name$_my_lib` + // would render to `foo_my_lib`. + ProtoNamingConvention = "python_proto_naming_convention" // DefaultVisibilty represents the directive that controls what visibility // labels are added to generated python targets. DefaultVisibilty = "python_default_visibility" @@ -121,6 +127,7 @@ const ( const ( packageNameNamingConventionSubstitution = "$package_name$" + protoNameNamingConventionSubstitution = "$proto_name$" distributionNameLabelConventionSubstitution = "$distribution_name$" ) @@ -182,6 +189,7 @@ type Config struct { libraryNamingConvention string binaryNamingConvention string testNamingConvention string + protoNamingConvention string defaultVisibility []string visibility []string testFilePattern []string @@ -220,6 +228,7 @@ func New( libraryNamingConvention: packageNameNamingConventionSubstitution, binaryNamingConvention: fmt.Sprintf("%s_bin", packageNameNamingConventionSubstitution), testNamingConvention: fmt.Sprintf("%s_test", packageNameNamingConventionSubstitution), + protoNamingConvention: fmt.Sprintf("%s_py_pb2", protoNameNamingConventionSubstitution), defaultVisibility: []string{fmt.Sprintf(DefaultVisibilityFmtString, "")}, visibility: []string{}, testFilePattern: strings.Split(DefaultTestFilePatternString, ","), @@ -255,6 +264,7 @@ func (c *Config) NewChild() *Config { libraryNamingConvention: c.libraryNamingConvention, binaryNamingConvention: c.binaryNamingConvention, testNamingConvention: c.testNamingConvention, + protoNamingConvention: c.protoNamingConvention, defaultVisibility: c.defaultVisibility, visibility: c.visibility, testFilePattern: c.testFilePattern, @@ -489,6 +499,17 @@ func (c *Config) RenderTestName(packageName string) string { return strings.ReplaceAll(c.testNamingConvention, packageNameNamingConventionSubstitution, packageName) } +// SetProtoNamingConvention sets the py_proto_library target naming convention. +func (c *Config) SetProtoNamingConvention(protoNamingConvention string) { + c.protoNamingConvention = protoNamingConvention +} + +// RenderProtoName returns the py_proto_library target name by performing all +// substitutions. +func (c *Config) RenderProtoName(protoName string) string { + return strings.ReplaceAll(c.protoNamingConvention, protoNameNamingConventionSubstitution, strings.TrimSuffix(protoName, "_proto")) +} + // AppendVisibility adds additional items to the target's visibility. func (c *Config) AppendVisibility(visibility string) { c.visibility = append(c.visibility, visibility) From aa602298a8bc2945ebf42d0820ec30bc974c9216 Mon Sep 17 00:00:00 2001 From: Jaemin Choi <1dotolee@gmail.com> Date: Mon, 21 Jul 2025 23:01:30 +0900 Subject: [PATCH 13/28] fix(pypi): expose pypi packages only common to all python versions (#3107) Closes #2921 For a single pypi repo with multiple python versions, `all_requirements` fail when a pypi package supports Python version A but not version B. In this case, the pypi package would be included only in requirements lock file for version A, not in one for version B. However, the failure occurs since the package is included in `all_requirements` even for Python version B. (Minimal reproduction: https://github.com/dotoleeoak/rules-python-2921-repro) This happens since `packages` parameter for `hub_repository` targets are including all packages across all requirement lock files. Instead of union of packages, intersection of packages for requirement files should be passed to `packages` and exposed to `all_requirements` macro, so that those packages are compatible with all Python versions. --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 2 + python/private/pypi/extension.bzl | 10 ++- tests/pypi/extension/extension_tests.bzl | 98 ++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d54d3a8881..74a4409cbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,8 @@ END_UNRELEASED_TEMPLATE * Multi-line python imports (e.g. with escaped newlines) are now correctly processed by Gazelle. * (toolchains) `local_runtime_repo` works with multiarch Debian with Python 3.8 ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)). +* (pypi) Expose pypi packages only common to all Python versions in `all_requirements` + ([#2921](https://github.com/bazel-contrib/rules_python/issues/2921)). {#v0-0-0-added} ### Added diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 2c1528e18d..096256e4be 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -601,7 +601,15 @@ You cannot use both the additive_build_content and additive_build_content_file a extra_aliases.setdefault(hub_name, {}) for whl_name, aliases in out.extra_aliases.items(): extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) - exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages) + if hub_name not in exposed_packages: + exposed_packages[hub_name] = out.exposed_packages + else: + intersection = {} + for pkg in out.exposed_packages: + if pkg not in exposed_packages[hub_name]: + continue + intersection[pkg] = None + exposed_packages[hub_name] = intersection whl_libraries.update(out.whl_libraries) # TODO @aignas 2024-04-05: how do we support different requirement diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 0303843e80..52e0e29cb0 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -285,6 +285,104 @@ def _test_simple_multiple_requirements(env): _tests.append(_test_simple_multiple_requirements) +def _test_simple_multiple_python_versions(env): + pypi = _parse_modules( + env, + module_ctx = _mock_mctx( + _mod( + name = "rules_python", + parse = [ + _parse( + hub_name = "pypi", + python_version = "3.15", + requirements_lock = "requirements_3_15.txt", + ), + _parse( + hub_name = "pypi", + python_version = "3.16", + requirements_lock = "requirements_3_16.txt", + ), + ], + ), + read = lambda x: { + "requirements_3_15.txt": """ +simple==0.0.1 --hash=sha256:deadbeef +old-package==0.0.1 --hash=sha256:deadbaaf +""", + "requirements_3_16.txt": """ +simple==0.0.2 --hash=sha256:deadb00f +new-package==0.0.1 --hash=sha256:deadb00f2 +""", + }[x], + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + "python_3_16_host": "unit_test_interpreter_target", + }, + minor_mapping = { + "3.15": "3.15.19", + "3.16": "3.16.9", + }, + ) + + pypi.exposed_packages().contains_exactly({"pypi": ["simple"]}) + pypi.hub_group_map().contains_exactly({"pypi": {}}) + pypi.hub_whl_map().contains_exactly({ + "pypi": { + "new_package": { + "pypi_316_new_package": [ + whl_config_setting( + version = "3.16", + ), + ], + }, + "old_package": { + "pypi_315_old_package": [ + whl_config_setting( + version = "3.15", + ), + ], + }, + "simple": { + "pypi_315_simple": [ + whl_config_setting( + version = "3.15", + ), + ], + "pypi_316_simple": [ + whl_config_setting( + version = "3.16", + ), + ], + }, + }, + }) + pypi.whl_libraries().contains_exactly({ + "pypi_315_old_package": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "old-package==0.0.1 --hash=sha256:deadbaaf", + }, + "pypi_315_simple": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef", + }, + "pypi_316_new_package": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "new-package==0.0.1 --hash=sha256:deadb00f2", + }, + "pypi_316_simple": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "simple==0.0.2 --hash=sha256:deadb00f", + }, + }) + pypi.whl_mods().contains_exactly({}) + +_tests.append(_test_simple_multiple_python_versions) + def _test_simple_with_markers(env): pypi = _parse_modules( env, From 5281261a97235099a65aaeb83b74b30d16409798 Mon Sep 17 00:00:00 2001 From: Jonathan Woodbury Date: Mon, 21 Jul 2025 20:33:25 -0400 Subject: [PATCH 14/28] fix: normalize stub_path in repl.bzl (#3104) When a REPL target is run from an external Bazel module, the `stub_path` can have path components in it (e.g. "/..") which get rejected by the `Rlocation()` function in `runfiles.py` for not being normalized. This commit normalizes the path before it's passed to `Rlocation()`. Fixes #3101 --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 2 ++ python/private/repl_template.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74a4409cbb..5ad48bee3f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,8 @@ END_UNRELEASED_TEMPLATE ([#3099](https://github.com/bazel-contrib/rules_python/issues/3099)). * (pypi) Expose pypi packages only common to all Python versions in `all_requirements` ([#2921](https://github.com/bazel-contrib/rules_python/issues/2921)). +* (repl) Normalize the path for the `REPL` stub to make it possible to use the + default stub template from outside `rules_python` ({gh-issue}`3101`). {#v0-0-0-added} ### Added diff --git a/python/private/repl_template.py b/python/private/repl_template.py index 37f4529fbe..dd8beb9784 100644 --- a/python/private/repl_template.py +++ b/python/private/repl_template.py @@ -5,7 +5,9 @@ from python.runfiles import runfiles -STUB_PATH = "%stub_path%" +# runfiles.py will reject paths which aren't normalized, which can happen when the REPL rules are +# used from a remote module. +STUB_PATH = os.path.normpath("%stub_path%") def start_repl(): From e6bba92d4a3de943c77a50834c727318870bef48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:34:19 +0900 Subject: [PATCH 15/28] build(deps): bump typing-extensions from 4.13.2 to 4.14.1 in /docs (#3094) [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=typing-extensions&package-manager=pip&previous-version=4.13.2&new-version=4.14.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- docs/requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 9e46724770..cb8900bf95 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -350,9 +350,9 @@ sphinxcontrib-serializinghtml==2.0.0 \ --hash=sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331 \ --hash=sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d # via sphinx -typing-extensions==4.13.2 \ - --hash=sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c \ - --hash=sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef +typing-extensions==4.14.1 \ + --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ + --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 # via # rules-python-docs (docs/pyproject.toml) # sphinx-autodoc2 From ab3e3f790788e119bfb362a60819727d7450f0f3 Mon Sep 17 00:00:00 2001 From: Alex Martani Date: Mon, 28 Jul 2025 21:29:18 -0700 Subject: [PATCH 16/28] fix(gazelle): Do not resolve absolute imports to sibling modules (#3106) Currently, gazelle allows absolute imports to be resolved to sibling modules: an `import foo` statement will resolve to a `foo.py` file in the same folder if such file exists. This seems to be a Python 2 behavior (ie. pre-`from __future__ import absolute_import`), and doesn't work on the current rules_python setup. This behavior is explicitly tested in the [siblings_import](https://github.com/bazel-contrib/rules_python/tree/cbe6d38d01c14de46d90ea717d0f2090117533fa/gazelle/python/testdata/sibling_imports) test case. However, recreating the exact same repository layout from this test case and running `bazel test //pkg:unit_test`, the test fails with the import failing. This PR adds a new directive, `gazelle:python_resolve_sibling_imports`, to allow disabling such behavior. The actual changes are in 3 places: - In `gazelle/python/target.go`, the directive is added to `if t.siblingSrcs.Contains(fileName) && fileName != filepath.Base(dep.Filepath)`, which is where the import is converted to a full absolute import if it matches a sibling file; - In `gazelle/python/generate.go`, the handling of `conftest.py` was dependent on this behavior (ie. it added a dependency on the module `conftest`, assuming that it would be resolved to the relative module). That was modified to compute the full absolute module path instead. - In `gazelle/python/resolve.go`, resolve relative imports even when using file generation mode. I also explicitly added `gazelle:python_resolve_sibling_imports true` to any test that breaks if the default value of this directive is changed to `false`. --- CHANGELOG.md | 4 ++ gazelle/README.md | 2 + gazelle/python/configure.go | 7 +++ gazelle/python/generate.go | 16 +++--- gazelle/python/resolve.go | 10 ++-- gazelle/python/target.go | 53 ++++++++++--------- .../testdata/annotation_include_dep/BUILD.in | 1 + .../testdata/annotation_include_dep/BUILD.out | 1 + .../with_conftest/BUILD.in | 1 + .../with_conftest/BUILD.out | 2 + .../testdata/naming_convention/BUILD.in | 1 + .../testdata/naming_convention/BUILD.out | 1 + .../python/testdata/sibling_imports/README.md | 12 ++++- .../testdata/sibling_imports/pkg/BUILD.in | 1 + .../testdata/sibling_imports/pkg/BUILD.out | 3 +- .../sibling_imports_disabled/BUILD.in | 2 + .../sibling_imports_disabled/BUILD.out | 18 +++++++ .../sibling_imports_disabled/README.md | 22 ++++++++ .../sibling_imports_disabled/WORKSPACE | 1 + .../testdata/sibling_imports_disabled/a.py | 1 + .../testdata/sibling_imports_disabled/b.py | 3 ++ .../sibling_imports_disabled/pkg/BUILD.in | 0 .../sibling_imports_disabled/pkg/BUILD.out | 27 ++++++++++ .../sibling_imports_disabled/pkg/__init__.py | 0 .../sibling_imports_disabled/pkg/a.py | 0 .../sibling_imports_disabled/pkg/b.py | 2 + .../sibling_imports_disabled/pkg/test_util.py | 2 + .../sibling_imports_disabled/pkg/typing.py | 1 + .../sibling_imports_disabled/pkg/unit_test.py | 5 ++ .../sibling_imports_disabled/test.yaml | 1 + .../sibling_imports_disabled/test_util.py | 1 + .../BUILD.in | 3 ++ .../BUILD.out | 22 ++++++++ .../README.md | 22 ++++++++ .../WORKSPACE | 1 + .../sibling_imports_disabled_file_mode/a.py | 1 + .../sibling_imports_disabled_file_mode/b.py | 3 ++ .../pkg/BUILD.in | 0 .../pkg/BUILD.out | 38 +++++++++++++ .../pkg/__init__.py | 0 .../pkg/a.py | 0 .../pkg/b.py | 2 + .../pkg/test_util.py | 2 + .../pkg/typing.py | 1 + .../pkg/unit_test.py | 5 ++ .../test.yaml | 1 + .../test_util.py | 1 + .../simple_test_with_conftest/BUILD.in | 2 + .../simple_test_with_conftest/BUILD.out | 2 + .../BUILD.in | 3 ++ .../BUILD.out | 29 ++++++++++ .../README.md | 4 ++ .../WORKSPACE | 1 + .../__init__.py | 3 ++ .../__test__.py | 12 +++++ .../bar/BUILD.in | 1 + .../bar/BUILD.out | 27 ++++++++++ .../bar/__init__.py | 3 ++ .../bar/__test__.py | 12 +++++ .../bar/bar.py | 2 + .../bar/conftest.py | 0 .../conftest.py | 0 .../foo.py | 2 + .../test.yaml | 4 ++ .../python/testdata/subdir_sources/BUILD.in | 1 + .../python/testdata/subdir_sources/BUILD.out | 1 + gazelle/pythonconfig/pythonconfig.go | 18 +++++++ 67 files changed, 388 insertions(+), 42 deletions(-) create mode 100644 gazelle/python/testdata/sibling_imports_disabled/BUILD.in create mode 100644 gazelle/python/testdata/sibling_imports_disabled/BUILD.out create mode 100644 gazelle/python/testdata/sibling_imports_disabled/README.md create mode 100644 gazelle/python/testdata/sibling_imports_disabled/WORKSPACE create mode 100644 gazelle/python/testdata/sibling_imports_disabled/a.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/b.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/BUILD.in create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/BUILD.out create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/__init__.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/a.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/b.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/test_util.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/typing.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/pkg/unit_test.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled/test.yaml create mode 100644 gazelle/python/testdata/sibling_imports_disabled/test_util.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/WORKSPACE create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/a.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/b.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/BUILD.in create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/BUILD.out create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/__init__.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/a.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/b.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/test_util.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/typing.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/unit_test.py create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/test.yaml create mode 100644 gazelle/python/testdata/sibling_imports_disabled_file_mode/test_util.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.in create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.out create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/README.md create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/WORKSPACE create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__init__.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__test__.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.in create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.out create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__init__.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__test__.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/bar.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/conftest.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/conftest.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/foo.py create mode 100644 gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/test.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ad48bee3f..c52481d52e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,6 +95,10 @@ END_UNRELEASED_TEMPLATE ([#2921](https://github.com/bazel-contrib/rules_python/issues/2921)). * (repl) Normalize the path for the `REPL` stub to make it possible to use the default stub template from outside `rules_python` ({gh-issue}`3101`). +* (gazelle) Fixes gazelle adding sibling module dependencies to resolve + absolute imports (Python 2's behavior without `absolute_import`). Previous + behavior can be restored using the directive + `# gazelle:python_resolve_sibling_imports true` {#v0-0-0-added} ### Added diff --git a/gazelle/README.md b/gazelle/README.md index 222c1171ab..8b088a4e70 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -228,6 +228,8 @@ Python-specific directives are as follows: | Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. | | [`# gazelle:python_generate_proto`](#directive-python_generate_proto) | `false` | | Controls whether to generate a `py_proto_library` for each `proto_library` in the package. By default we load this rule from the `@protobuf` repository; use `gazelle:map_kind` if you need to load this from somewhere else. | +| `# gazelle:python_resolve_sibling_imports` | `false` | +| Allows absolute imports to be resolved to sibling modules (Python 2's behavior without `absolute_import`). | #### Directive: `python_root`: diff --git a/gazelle/python/configure.go b/gazelle/python/configure.go index 079f1d84d4..13ba6477cd 100644 --- a/gazelle/python/configure.go +++ b/gazelle/python/configure.go @@ -72,6 +72,7 @@ func (py *Configurer) KnownDirectives() []string { pythonconfig.GeneratePyiDeps, pythonconfig.ExperimentalAllowRelativeImports, pythonconfig.GenerateProto, + pythonconfig.PythonResolveSiblingImports, } } @@ -247,6 +248,12 @@ func (py *Configurer) Configure(c *config.Config, rel string, f *rule.File) { log.Fatal(err) } config.SetGenerateProto(v) + case pythonconfig.PythonResolveSiblingImports: + v, err := strconv.ParseBool(strings.TrimSpace(d.Value)) + if err != nil { + log.Fatal(err) + } + config.SetResolveSiblingImports(v) } } diff --git a/gazelle/python/generate.go b/gazelle/python/generate.go index 5b6ba79d69..a180ec527d 100644 --- a/gazelle/python/generate.go +++ b/gazelle/python/generate.go @@ -259,7 +259,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes fqTarget.String(), actualPyBinaryKind, err) continue } - pyBinary := newTargetBuilder(pyBinaryKind, pyBinaryTargetName, pythonProjectRoot, args.Rel, pyFileNames). + pyBinary := newTargetBuilder(pyBinaryKind, pyBinaryTargetName, pythonProjectRoot, args.Rel, pyFileNames, cfg.ResolveSiblingImports()). addVisibility(visibility). addSrc(filename). addModuleDependencies(mainModules[filename]). @@ -301,7 +301,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes collisionErrors.Add(err) } - pyLibrary := newTargetBuilder(pyLibraryKind, pyLibraryTargetName, pythonProjectRoot, args.Rel, pyFileNames). + pyLibrary := newTargetBuilder(pyLibraryKind, pyLibraryTargetName, pythonProjectRoot, args.Rel, pyFileNames, cfg.ResolveSiblingImports()). addVisibility(visibility). addSrcs(srcs). addModuleDependencies(allDeps). @@ -354,7 +354,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes collisionErrors.Add(err) } - pyBinaryTarget := newTargetBuilder(pyBinaryKind, pyBinaryTargetName, pythonProjectRoot, args.Rel, pyFileNames). + pyBinaryTarget := newTargetBuilder(pyBinaryKind, pyBinaryTargetName, pythonProjectRoot, args.Rel, pyFileNames, cfg.ResolveSiblingImports()). setMain(pyBinaryEntrypointFilename). addVisibility(visibility). addSrc(pyBinaryEntrypointFilename). @@ -387,7 +387,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes collisionErrors.Add(err) } - conftestTarget := newTargetBuilder(pyLibraryKind, conftestTargetname, pythonProjectRoot, args.Rel, pyFileNames). + conftestTarget := newTargetBuilder(pyLibraryKind, conftestTargetname, pythonProjectRoot, args.Rel, pyFileNames, cfg.ResolveSiblingImports()). addSrc(conftestFilename). addModuleDependencies(deps). addResolvedDependencies(annotations.includeDeps). @@ -419,7 +419,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes fqTarget.String(), actualPyTestKind, err, pythonconfig.TestNamingConvention) collisionErrors.Add(err) } - return newTargetBuilder(pyTestKind, pyTestTargetName, pythonProjectRoot, args.Rel, pyFileNames). + return newTargetBuilder(pyTestKind, pyTestTargetName, pythonProjectRoot, args.Rel, pyFileNames, cfg.ResolveSiblingImports()). addSrcs(srcs). addModuleDependencies(deps). addResolvedDependencies(annotations.includeDeps). @@ -476,7 +476,7 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes for _, pyTestTarget := range pyTestTargets { if conftest != nil { - conftestModule := Module{Name: strings.TrimSuffix(conftestFilename, ".py")} + conftestModule := Module{Name: importSpecFromSrc(pythonProjectRoot, args.Rel, conftestFilename).Imp} if pyTestTarget.annotations.includePytestConftest == nil { // unset; default behavior pyTestTarget.addModuleDependency(conftestModule) @@ -605,7 +605,7 @@ func generateProtoLibraries(args language.GenerateArgs, cfg *pythonconfig.Config pyProtoLibraryName = ruleName } - pyProtoLibrary := newTargetBuilder(pyProtoLibraryKind, pyProtoLibraryName, pythonProjectRoot, args.Rel, &emptySiblings). + pyProtoLibrary := newTargetBuilder(pyProtoLibraryKind, pyProtoLibraryName, pythonProjectRoot, args.Rel, &emptySiblings, false). addVisibility(visibility). addResolvedDependency(":" + protoRuleName). generateImportsAttribute().build() @@ -622,7 +622,7 @@ func generateProtoLibraries(args language.GenerateArgs, cfg *pythonconfig.Config continue } - emptyRule := newTargetBuilder(pyProtoLibraryKind, ruleName, pythonProjectRoot, args.Rel, &emptySiblings).build() + emptyRule := newTargetBuilder(pyProtoLibraryKind, ruleName, pythonProjectRoot, args.Rel, &emptySiblings, false).build() res.Empty = append(res.Empty, emptyRule) } diff --git a/gazelle/python/resolve.go b/gazelle/python/resolve.go index 0dd80841d4..cc57180a49 100644 --- a/gazelle/python/resolve.go +++ b/gazelle/python/resolve.go @@ -164,8 +164,6 @@ func (py *Resolver) Resolve( modules := modulesRaw.(*treeset.Set) it := modules.Iterator() explainDependency := os.Getenv("EXPLAIN_DEPENDENCY") - // Resolve relative paths for package generation - isPackageGeneration := !cfg.PerFileGeneration() && !cfg.CoarseGrainedGeneration() hasFatalError := false MODULES_LOOP: for it.Next() { @@ -173,7 +171,7 @@ func (py *Resolver) Resolve( moduleName := mod.Name // Transform relative imports `.` or `..foo.bar` into the package path from root. if strings.HasPrefix(mod.From, ".") { - if !cfg.ExperimentalAllowRelativeImports() || !isPackageGeneration { + if !cfg.ExperimentalAllowRelativeImports() { continue MODULES_LOOP } @@ -210,9 +208,9 @@ func (py *Resolver) Resolve( baseParts = pkgParts[:len(pkgParts)-(relativeDepth-1)] } // Build absolute module path - absParts := append([]string{}, baseParts...) // base path - absParts = append(absParts, fromParts...) // subpath from 'from' - absParts = append(absParts, imported) // actual imported symbol + absParts := append([]string{}, baseParts...) // base path + absParts = append(absParts, fromParts...) // subpath from 'from' + absParts = append(absParts, imported) // actual imported symbol moduleName = strings.Join(absParts, ".") } diff --git a/gazelle/python/target.go b/gazelle/python/target.go index 6e6c3f4b14..3fe5819e00 100644 --- a/gazelle/python/target.go +++ b/gazelle/python/target.go @@ -25,34 +25,36 @@ import ( // targetBuilder builds targets to be generated by Gazelle. type targetBuilder struct { - kind string - name string - pythonProjectRoot string - bzlPackage string - srcs *treeset.Set - siblingSrcs *treeset.Set - deps *treeset.Set - resolvedDeps *treeset.Set - visibility *treeset.Set - main *string - imports []string - testonly bool - annotations *annotations + kind string + name string + pythonProjectRoot string + bzlPackage string + srcs *treeset.Set + siblingSrcs *treeset.Set + deps *treeset.Set + resolvedDeps *treeset.Set + visibility *treeset.Set + main *string + imports []string + testonly bool + annotations *annotations + resolveSiblingImports bool } // newTargetBuilder constructs a new targetBuilder. -func newTargetBuilder(kind, name, pythonProjectRoot, bzlPackage string, siblingSrcs *treeset.Set) *targetBuilder { +func newTargetBuilder(kind, name, pythonProjectRoot, bzlPackage string, siblingSrcs *treeset.Set, resolveSiblingImports bool) *targetBuilder { return &targetBuilder{ - kind: kind, - name: name, - pythonProjectRoot: pythonProjectRoot, - bzlPackage: bzlPackage, - srcs: treeset.NewWith(godsutils.StringComparator), - siblingSrcs: siblingSrcs, - deps: treeset.NewWith(moduleComparator), - resolvedDeps: treeset.NewWith(godsutils.StringComparator), - visibility: treeset.NewWith(godsutils.StringComparator), - annotations: new(annotations), + kind: kind, + name: name, + pythonProjectRoot: pythonProjectRoot, + bzlPackage: bzlPackage, + srcs: treeset.NewWith(godsutils.StringComparator), + siblingSrcs: siblingSrcs, + deps: treeset.NewWith(moduleComparator), + resolvedDeps: treeset.NewWith(godsutils.StringComparator), + visibility: treeset.NewWith(godsutils.StringComparator), + annotations: new(annotations), + resolveSiblingImports: resolveSiblingImports, } } @@ -77,7 +79,7 @@ func (t *targetBuilder) addModuleDependency(dep Module) *targetBuilder { if dep.From != "" { fileName = dep.From + ".py" } - if t.siblingSrcs.Contains(fileName) && fileName != filepath.Base(dep.Filepath) { + if t.resolveSiblingImports && t.siblingSrcs.Contains(fileName) && fileName != filepath.Base(dep.Filepath) { // importing another module from the same package, converting to absolute imports to make // dependency resolution easier dep.Name = importSpecFromSrc(t.pythonProjectRoot, t.bzlPackage, fileName).Imp @@ -138,7 +140,6 @@ func (t *targetBuilder) setAnnotations(val annotations) *targetBuilder { return t } - // generateImportsAttribute generates the imports attribute. // These are a list of import directories to be added to the PYTHONPATH. In our // case, the value we add is on Bazel sub-packages to be able to perform imports diff --git a/gazelle/python/testdata/annotation_include_dep/BUILD.in b/gazelle/python/testdata/annotation_include_dep/BUILD.in index af2c2cea4b..5131712aca 100644 --- a/gazelle/python/testdata/annotation_include_dep/BUILD.in +++ b/gazelle/python/testdata/annotation_include_dep/BUILD.in @@ -1 +1,2 @@ # gazelle:python_generation_mode file +# gazelle:python_resolve_sibling_imports true diff --git a/gazelle/python/testdata/annotation_include_dep/BUILD.out b/gazelle/python/testdata/annotation_include_dep/BUILD.out index 1cff8f4676..412bf456f5 100644 --- a/gazelle/python/testdata/annotation_include_dep/BUILD.out +++ b/gazelle/python/testdata/annotation_include_dep/BUILD.out @@ -1,6 +1,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") # gazelle:python_generation_mode file +# gazelle:python_resolve_sibling_imports true py_library( name = "__init__", diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.in b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.in index e69de29bb2..5c25b0d5a6 100644 --- a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.in +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_resolve_sibling_imports true diff --git a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out index 60695352ca..52b915208e 100644 --- a/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out +++ b/gazelle/python/testdata/annotation_include_pytest_conftest/with_conftest/BUILD.out @@ -1,5 +1,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") +# gazelle:python_resolve_sibling_imports true + py_binary( name = "binary", srcs = ["binary.py"], diff --git a/gazelle/python/testdata/naming_convention/BUILD.in b/gazelle/python/testdata/naming_convention/BUILD.in index 7517848a92..fee53ba7ff 100644 --- a/gazelle/python/testdata/naming_convention/BUILD.in +++ b/gazelle/python/testdata/naming_convention/BUILD.in @@ -1,3 +1,4 @@ # gazelle:python_library_naming_convention my_$package_name$_library # gazelle:python_binary_naming_convention my_$package_name$_binary # gazelle:python_test_naming_convention my_$package_name$_test +# gazelle:python_resolve_sibling_imports true diff --git a/gazelle/python/testdata/naming_convention/BUILD.out b/gazelle/python/testdata/naming_convention/BUILD.out index e2f067489c..7392cfeb35 100644 --- a/gazelle/python/testdata/naming_convention/BUILD.out +++ b/gazelle/python/testdata/naming_convention/BUILD.out @@ -3,6 +3,7 @@ load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") # gazelle:python_library_naming_convention my_$package_name$_library # gazelle:python_binary_naming_convention my_$package_name$_binary # gazelle:python_test_naming_convention my_$package_name$_test +# gazelle:python_resolve_sibling_imports true py_library( name = "my_naming_convention_library", diff --git a/gazelle/python/testdata/sibling_imports/README.md b/gazelle/python/testdata/sibling_imports/README.md index e59be07634..d21a671b1c 100644 --- a/gazelle/python/testdata/sibling_imports/README.md +++ b/gazelle/python/testdata/sibling_imports/README.md @@ -1,3 +1,13 @@ # Sibling imports -This test case asserts that imports from sibling modules are resolved correctly. It covers 3 different types of imports in `pkg/unit_test.py` \ No newline at end of file +This test case asserts that imports from sibling modules are resolved correctly +when the `python_resolve_sibling_imports` directive is enabled (default +behavior). It covers 3 different types of imports in `pkg/unit_test.py`: + +- `import a` - resolves to the sibling `a.py` in the same package +- `import test_util` - resolves to the sibling `test_util.py` in the same + package +- `from b import run` - resolves to the sibling `b.py` in the same package + +When sibling imports are enabled, we allow them to be satisfied by sibling +modules (ie. modules in the same package). diff --git a/gazelle/python/testdata/sibling_imports/pkg/BUILD.in b/gazelle/python/testdata/sibling_imports/pkg/BUILD.in index e69de29bb2..5c25b0d5a6 100644 --- a/gazelle/python/testdata/sibling_imports/pkg/BUILD.in +++ b/gazelle/python/testdata/sibling_imports/pkg/BUILD.in @@ -0,0 +1 @@ +# gazelle:python_resolve_sibling_imports true diff --git a/gazelle/python/testdata/sibling_imports/pkg/BUILD.out b/gazelle/python/testdata/sibling_imports/pkg/BUILD.out index cae6c3f17a..e8c13098c2 100644 --- a/gazelle/python/testdata/sibling_imports/pkg/BUILD.out +++ b/gazelle/python/testdata/sibling_imports/pkg/BUILD.out @@ -1,5 +1,7 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") +# gazelle:python_resolve_sibling_imports true + py_library( name = "pkg", srcs = [ @@ -23,4 +25,3 @@ py_test( ":test_util", ], ) - diff --git a/gazelle/python/testdata/sibling_imports_disabled/BUILD.in b/gazelle/python/testdata/sibling_imports_disabled/BUILD.in new file mode 100644 index 0000000000..9509fd9727 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/BUILD.in @@ -0,0 +1,2 @@ +# gazelle:python_resolve_sibling_imports false +# gazelle:experimental_allow_relative_imports true diff --git a/gazelle/python/testdata/sibling_imports_disabled/BUILD.out b/gazelle/python/testdata/sibling_imports_disabled/BUILD.out new file mode 100644 index 0000000000..7568f38f50 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/BUILD.out @@ -0,0 +1,18 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +# gazelle:python_resolve_sibling_imports false +# gazelle:experimental_allow_relative_imports true + +py_library( + name = "sibling_imports_disabled", + srcs = [ + "a.py", + "b.py", + ], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "test_util", + srcs = ["test_util.py"], +) diff --git a/gazelle/python/testdata/sibling_imports_disabled/README.md b/gazelle/python/testdata/sibling_imports_disabled/README.md new file mode 100644 index 0000000000..d534a44bf1 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/README.md @@ -0,0 +1,22 @@ +# Sibling imports disabled + +This test case asserts that imports from sibling modules are NOT resolved as +absolute imports when the `python_resolve_sibling_imports` directive is +disabled. It covers different types of imports in `pkg/unit_test.py`: + +- `import a` - resolves to the root-level `a.py` instead of the sibling + `pkg/a.py` +- `from typing import Iterable` - resolves to the stdlib `typing` module + (not the sibling `typing.py`). +- `from .b import run` / `from .typing import A` - resolves to the sibling + `pkg/b.py` / `pkg/typing.py` (with + `gazelle:experimental_allow_relative_imports` enabled) +- `import test_util` - resolves to the root-level `test_util.py` instead of + the sibling `pkg/test_util.py` +- `from b import run` - resolves to the root-level `b.py` instead of the + sibling `pkg/b.py` + +When sibling imports are disabled with +`# gazelle:python_resolve_sibling_imports false`, the imports remain as-is +and follow standard Python resolution rules where absolute imports can't refer +to sibling modules. diff --git a/gazelle/python/testdata/sibling_imports_disabled/WORKSPACE b/gazelle/python/testdata/sibling_imports_disabled/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/sibling_imports_disabled/a.py b/gazelle/python/testdata/sibling_imports_disabled/a.py new file mode 100644 index 0000000000..fad4fb1ff9 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/a.py @@ -0,0 +1 @@ +# Root level a.py file for testing disabled sibling imports diff --git a/gazelle/python/testdata/sibling_imports_disabled/b.py b/gazelle/python/testdata/sibling_imports_disabled/b.py new file mode 100644 index 0000000000..a5eafc436f --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/b.py @@ -0,0 +1,3 @@ +# Root level b.py file for testing disabled sibling imports +def run(): + pass diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/BUILD.in b/gazelle/python/testdata/sibling_imports_disabled/pkg/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/BUILD.out b/gazelle/python/testdata/sibling_imports_disabled/pkg/BUILD.out new file mode 100644 index 0000000000..e778ce1076 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/pkg/BUILD.out @@ -0,0 +1,27 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "pkg", + srcs = [ + "__init__.py", + "a.py", + "b.py", + "typing.py", + ], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "test_util", + srcs = ["test_util.py"], + deps = [":pkg"], +) + +py_test( + name = "unit_test", + srcs = ["unit_test.py"], + deps = [ + "//:sibling_imports_disabled", + "//:test_util", + ], +) diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/__init__.py b/gazelle/python/testdata/sibling_imports_disabled/pkg/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/a.py b/gazelle/python/testdata/sibling_imports_disabled/pkg/a.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/b.py b/gazelle/python/testdata/sibling_imports_disabled/pkg/b.py new file mode 100644 index 0000000000..d04d423678 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/pkg/b.py @@ -0,0 +1,2 @@ +def run(): + pass diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/test_util.py b/gazelle/python/testdata/sibling_imports_disabled/pkg/test_util.py new file mode 100644 index 0000000000..01cc15da86 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/pkg/test_util.py @@ -0,0 +1,2 @@ +from .b import run +from .typing import A diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/typing.py b/gazelle/python/testdata/sibling_imports_disabled/pkg/typing.py new file mode 100644 index 0000000000..76f516f79f --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/pkg/typing.py @@ -0,0 +1 @@ +A = 1 diff --git a/gazelle/python/testdata/sibling_imports_disabled/pkg/unit_test.py b/gazelle/python/testdata/sibling_imports_disabled/pkg/unit_test.py new file mode 100644 index 0000000000..3c551a1cb1 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/pkg/unit_test.py @@ -0,0 +1,5 @@ +from typing import Iterable + +import a +import test_util +from b import run diff --git a/gazelle/python/testdata/sibling_imports_disabled/test.yaml b/gazelle/python/testdata/sibling_imports_disabled/test.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/test.yaml @@ -0,0 +1 @@ +--- diff --git a/gazelle/python/testdata/sibling_imports_disabled/test_util.py b/gazelle/python/testdata/sibling_imports_disabled/test_util.py new file mode 100644 index 0000000000..f5fa1b34ea --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled/test_util.py @@ -0,0 +1 @@ +# Root level test_util.py file for testing disabled sibling imports diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in new file mode 100644 index 0000000000..04494394c7 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in @@ -0,0 +1,3 @@ +# gazelle:python_generation_mode file +# gazelle:python_resolve_sibling_imports false +# gazelle:experimental_allow_relative_imports true diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out new file mode 100644 index 0000000000..da53e14864 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out @@ -0,0 +1,22 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +# gazelle:python_generation_mode file +# gazelle:python_resolve_sibling_imports false +# gazelle:experimental_allow_relative_imports true + +py_library( + name = "a", + srcs = ["a.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "b", + srcs = ["b.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "test_util", + srcs = ["test_util.py"], +) diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md b/gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md new file mode 100644 index 0000000000..0bfbcffb58 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md @@ -0,0 +1,22 @@ +# Sibling imports disabled (file generation mode) + +This test case asserts that imports from sibling modules are NOT resolved as +absolute imports when the `python_resolve_sibling_imports` directive is +disabled. It covers different types of imports in `pkg/unit_test.py`: + +- `import a` - resolves to the root-level `a.py` instead of the sibling + `pkg/a.py` +- `from typing import Iterable` - resolves to the stdlib `typing` module + (not the sibling `typing.py`). +- `from .b import run` / `from .typing import A` - resolves to the sibling + `pkg/b.py` / `pkg/typing.py` (with + `gazelle:experimental_allow_relative_imports` enabled) +- `import test_util` - resolves to the root-level `test_util.py` instead of + the sibling `pkg/test_util.py` +- `from b import run` - resolves to the root-level `b.py` instead of the + sibling `pkg/b.py` + +When sibling imports are disabled with +`# gazelle:python_resolve_sibling_imports false`, the imports remain as-is +and follow standard Python resolution rules where absolute imports can't refer +to sibling modules. diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/WORKSPACE b/gazelle/python/testdata/sibling_imports_disabled_file_mode/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/a.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/a.py new file mode 100644 index 0000000000..fad4fb1ff9 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/a.py @@ -0,0 +1 @@ +# Root level a.py file for testing disabled sibling imports diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/b.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/b.py new file mode 100644 index 0000000000..a5eafc436f --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/b.py @@ -0,0 +1,3 @@ +# Root level b.py file for testing disabled sibling imports +def run(): + pass diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/BUILD.in b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/BUILD.in new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/BUILD.out b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/BUILD.out new file mode 100644 index 0000000000..ab161e135f --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/BUILD.out @@ -0,0 +1,38 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "a", + srcs = ["a.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "b", + srcs = ["b.py"], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "typing", + srcs = ["typing.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "test_util", + srcs = ["test_util.py"], + deps = [ + ":b", + ":typing", + ], +) + +py_test( + name = "unit_test", + srcs = ["unit_test.py"], + deps = [ + "//:a", + "//:b", + "//:test_util", + ], +) diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/__init__.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/a.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/a.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/b.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/b.py new file mode 100644 index 0000000000..d04d423678 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/b.py @@ -0,0 +1,2 @@ +def run(): + pass diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/test_util.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/test_util.py new file mode 100644 index 0000000000..01cc15da86 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/test_util.py @@ -0,0 +1,2 @@ +from .b import run +from .typing import A diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/typing.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/typing.py new file mode 100644 index 0000000000..76f516f79f --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/typing.py @@ -0,0 +1 @@ +A = 1 diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/unit_test.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/unit_test.py new file mode 100644 index 0000000000..3c551a1cb1 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/pkg/unit_test.py @@ -0,0 +1,5 @@ +from typing import Iterable + +import a +import test_util +from b import run diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/test.yaml b/gazelle/python/testdata/sibling_imports_disabled_file_mode/test.yaml new file mode 100644 index 0000000000..ed97d539c0 --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/test.yaml @@ -0,0 +1 @@ +--- diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/test_util.py b/gazelle/python/testdata/sibling_imports_disabled_file_mode/test_util.py new file mode 100644 index 0000000000..f5fa1b34ea --- /dev/null +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/test_util.py @@ -0,0 +1 @@ +# Root level test_util.py file for testing disabled sibling imports diff --git a/gazelle/python/testdata/simple_test_with_conftest/BUILD.in b/gazelle/python/testdata/simple_test_with_conftest/BUILD.in index 3f2beb3147..6dfab75442 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/BUILD.in +++ b/gazelle/python/testdata/simple_test_with_conftest/BUILD.in @@ -1 +1,3 @@ load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_resolve_sibling_imports true diff --git a/gazelle/python/testdata/simple_test_with_conftest/BUILD.out b/gazelle/python/testdata/simple_test_with_conftest/BUILD.out index 18079bf2f4..62e1c550e6 100644 --- a/gazelle/python/testdata/simple_test_with_conftest/BUILD.out +++ b/gazelle/python/testdata/simple_test_with_conftest/BUILD.out @@ -1,5 +1,7 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") +# gazelle:python_resolve_sibling_imports true + py_library( name = "simple_test_with_conftest", srcs = [ diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.in b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.in new file mode 100644 index 0000000000..f8a40fe26c --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.in @@ -0,0 +1,3 @@ +load("@rules_python//python:defs.bzl", "py_library") + +# gazelle:python_resolve_sibling_imports false diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.out b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.out new file mode 100644 index 0000000000..b5a7066aff --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/BUILD.out @@ -0,0 +1,29 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +# gazelle:python_resolve_sibling_imports false + +py_library( + name = "simple_test_with_conftest_sibling_imports_disabled", + srcs = [ + "__init__.py", + "foo.py", + ], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "simple_test_with_conftest_sibling_imports_disabled_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [ + ":conftest", + ":simple_test_with_conftest_sibling_imports_disabled", + ], +) diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/README.md b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/README.md new file mode 100644 index 0000000000..98793c23de --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/README.md @@ -0,0 +1,4 @@ +# Simple test with conftest.py (sibling imports disable) + +This test case asserts that a simple `py_test` is generated as expected when a +`conftest.py` is present with sibling imports disabled. diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/WORKSPACE b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/WORKSPACE new file mode 100644 index 0000000000..faff6af87a --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/WORKSPACE @@ -0,0 +1 @@ +# This is a Bazel workspace for the Gazelle test data. diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__init__.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__init__.py new file mode 100644 index 0000000000..6a49193fe4 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__init__.py @@ -0,0 +1,3 @@ +from foo import foo + +_ = foo diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__test__.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__test__.py new file mode 100644 index 0000000000..d6085a41b4 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/__test__.py @@ -0,0 +1,12 @@ +import unittest + +from __init__ import foo + + +class FooTest(unittest.TestCase): + def test_foo(self): + self.assertEqual("foo", foo()) + + +if __name__ == "__main__": + unittest.main() diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.in b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.in new file mode 100644 index 0000000000..3f2beb3147 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.in @@ -0,0 +1 @@ +load("@rules_python//python:defs.bzl", "py_library") diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.out b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.out new file mode 100644 index 0000000000..ef8591f199 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/BUILD.out @@ -0,0 +1,27 @@ +load("@rules_python//python:defs.bzl", "py_library", "py_test") + +py_library( + name = "bar", + srcs = [ + "__init__.py", + "bar.py", + ], + visibility = ["//:__subpackages__"], +) + +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "bar_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [ + ":conftest", + "//:simple_test_with_conftest_sibling_imports_disabled", + ], +) diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__init__.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__init__.py new file mode 100644 index 0000000000..0c59205559 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__init__.py @@ -0,0 +1,3 @@ +from bar import bar + +_ = bar diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__test__.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__test__.py new file mode 100644 index 0000000000..c3d4734eed --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/__test__.py @@ -0,0 +1,12 @@ +import unittest + +from __init__ import bar + + +class BarTest(unittest.TestCase): + def test_bar(self): + self.assertEqual("bar", bar()) + + +if __name__ == "__main__": + unittest.main() diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/bar.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/bar.py new file mode 100644 index 0000000000..ee70a51f03 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/bar.py @@ -0,0 +1,2 @@ +def bar(): + return "bar" diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/conftest.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/bar/conftest.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/conftest.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/conftest.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/foo.py b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/foo.py new file mode 100644 index 0000000000..cf68624419 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/foo.py @@ -0,0 +1,2 @@ +def foo(): + return "foo" diff --git a/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/test.yaml b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/test.yaml new file mode 100644 index 0000000000..8071ef4094 --- /dev/null +++ b/gazelle/python/testdata/simple_test_with_conftest_sibling_imports_disabled/test.yaml @@ -0,0 +1,4 @@ + +--- +expect: + exit_code: 0 diff --git a/gazelle/python/testdata/subdir_sources/BUILD.in b/gazelle/python/testdata/subdir_sources/BUILD.in index adfdefdc8a..e8f3827bd2 100644 --- a/gazelle/python/testdata/subdir_sources/BUILD.in +++ b/gazelle/python/testdata/subdir_sources/BUILD.in @@ -1 +1,2 @@ # gazelle:python_generation_mode project +# gazelle:python_resolve_sibling_imports true diff --git a/gazelle/python/testdata/subdir_sources/BUILD.out b/gazelle/python/testdata/subdir_sources/BUILD.out index 5d77890d4f..5b96ad7576 100644 --- a/gazelle/python/testdata/subdir_sources/BUILD.out +++ b/gazelle/python/testdata/subdir_sources/BUILD.out @@ -2,6 +2,7 @@ load("@rules_python//python:defs.bzl", "py_binary") # gazelle:python_generation_mode project +# gazelle:python_resolve_sibling_imports true py_binary( name = "subdir_sources_bin", diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index 001fd334a4..b3d56591ee 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -107,6 +107,11 @@ const ( // GenerateProto represents the directive that controls whether to generate // python_generate_proto targets. GenerateProto = "python_generate_proto" + // PythonResolveSiblingImports represents the directive that controls whether + // absolute imports can be solved to sibling modules. When enabled, imports + // like "import a" can be resolved to sibling modules. When disabled, they + // can only be resolved as an absolute import. + PythonResolveSiblingImports = "python_resolve_sibling_imports" ) // GenerationModeType represents one of the generation modes for the Python @@ -198,6 +203,7 @@ type Config struct { experimentalAllowRelativeImports bool generatePyiDeps bool generateProto bool + resolveSiblingImports bool } type LabelNormalizationType int @@ -237,6 +243,7 @@ func New( experimentalAllowRelativeImports: false, generatePyiDeps: false, generateProto: false, + resolveSiblingImports: false, } } @@ -273,6 +280,7 @@ func (c *Config) NewChild() *Config { experimentalAllowRelativeImports: c.experimentalAllowRelativeImports, generatePyiDeps: c.generatePyiDeps, generateProto: c.generateProto, + resolveSiblingImports: c.resolveSiblingImports, } } @@ -592,6 +600,16 @@ func (c *Config) GenerateProto() bool { return c.generateProto } +// SetResolveSiblingImports sets whether absolute imports can be resolved to sibling modules. +func (c *Config) SetResolveSiblingImports(resolveSiblingImports bool) { + c.resolveSiblingImports = resolveSiblingImports +} + +// ResolveSiblingImports returns whether absolute imports can be resolved to sibling modules. +func (c *Config) ResolveSiblingImports() bool { + return c.resolveSiblingImports +} + // FormatThirdPartyDependency returns a label to a third-party dependency performing all formating and normalization. func (c *Config) FormatThirdPartyDependency(repositoryName string, distributionName string) label.Label { conventionalDistributionName := strings.ReplaceAll(c.labelConvention, distributionNameLabelConventionSubstitution, distributionName) From 0d0ab5cbf6ebd4c7a7c1be44c74df56be12185c9 Mon Sep 17 00:00:00 2001 From: Jaemin Choi <1dotolee@gmail.com> Date: Sat, 2 Aug 2025 08:07:35 +0900 Subject: [PATCH 17/28] fix(pypi): show overridden index urls in pypi download error (#3130) Closes #2985 Suppose invalid `experimental_index_url_overrides` in `pip.parse` is set like below. ```bzl pip.parse( experimental_index_url = "https://pypi.org/simple", experimental_index_url_overrides = {"mypy": "https://invalid.com"}, hub_name = "pypi", requirements_lock = "//:requirements_lock.txt", ) ``` It fails as follows, showing only "pypi.org" as pypi index url, not "invalid.com" for for `mypy` package. ``` Error in fail: Failed to download metadata for ["mypy"] for from urls: ["https://pypi.org/simple"]. If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=["mypy"]' to your 'pip.parse' call. ``` To show overridden url for each package, show url of each package that has been failed to download metadata. The error message with this PR is like below. ``` Error in fail: Failed to download metadata of the following packages from urls: { "mypy": "https://invalid.com", } If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=["mypy"]' to your 'pip.parse' call. ``` --- CHANGELOG.md | 2 ++ python/private/pypi/simpleapi_download.bzl | 26 +++++++++++------ .../simpleapi_download_tests.bzl | 28 +++++++++++++++---- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c52481d52e..d21ebc7d36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,6 +99,8 @@ END_UNRELEASED_TEMPLATE absolute imports (Python 2's behavior without `absolute_import`). Previous behavior can be restored using the directive `# gazelle:python_resolve_sibling_imports true` +* (pypi) Show overridden index URL of packages when downloading metadata have failed. + ([#2985](https://github.com/bazel-contrib/rules_python/issues/2985)). {#v0-0-0-added} ### Added diff --git a/python/private/pypi/simpleapi_download.bzl b/python/private/pypi/simpleapi_download.bzl index a3ba9691cd..52ff02a178 100644 --- a/python/private/pypi/simpleapi_download.bzl +++ b/python/private/pypi/simpleapi_download.bzl @@ -128,16 +128,24 @@ def simpleapi_download( failed_sources = [pkg for pkg in attr.sources if pkg not in found_on_index] if failed_sources: + pkg_index_urls = { + pkg: index_url_overrides.get( + normalize_name(pkg), + index_urls, + ) + for pkg in failed_sources + } + _fail( - "\n".join([ - "Failed to download metadata for {} for from urls: {}.".format( - failed_sources, - index_urls, - ), - "If you would like to skip downloading metadata for these packages please add 'simpleapi_skip={}' to your 'pip.parse' call.".format( - render.list(failed_sources), - ), - ]), + """ +Failed to download metadata of the following packages from urls: +{pkg_index_urls} + +If you would like to skip downloading metadata for these packages please add 'simpleapi_skip={failed_sources}' to your 'pip.parse' call. +""".format( + pkg_index_urls = render.dict(pkg_index_urls), + failed_sources = render.list(failed_sources), + ), ) return None diff --git a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl index a96815c12c..8dc307235a 100644 --- a/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl +++ b/tests/pypi/simpleapi_download/simpleapi_download_tests.bzl @@ -87,6 +87,11 @@ def _test_fail(env): output = "", success = False, ) + if "bar" in url: + return struct( + output = "", + success = False, + ) else: return struct( output = "data from {}".format(url), @@ -99,7 +104,9 @@ def _test_fail(env): report_progress = lambda _: None, ), attr = struct( - index_url_overrides = {}, + index_url_overrides = { + "foo": "invalid", + }, index_url = "main", extra_index_urls = ["extra"], sources = ["foo", "bar", "baz"], @@ -112,16 +119,25 @@ def _test_fail(env): ) env.expect.that_collection(fails).contains_exactly([ - """\ -Failed to download metadata for ["foo"] for from urls: ["main", "extra"]. -If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=["foo"]' to your 'pip.parse' call.\ + """ +Failed to download metadata of the following packages from urls: +{ + "foo": "invalid", + "bar": ["main", "extra"], +} + +If you would like to skip downloading metadata for these packages please add 'simpleapi_skip=[ + "foo", + "bar", +]' to your 'pip.parse' call. """, ]) env.expect.that_collection(calls).contains_exactly([ - "extra/foo/", + "invalid/foo/", "main/bar/", "main/baz/", - "main/foo/", + "invalid/foo/", + "extra/bar/", ]) _tests.append(_test_fail) From 2c53bf6968e170850237cbd0779337b093f4f94f Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Fri, 1 Aug 2025 16:08:21 -0700 Subject: [PATCH 18/28] chore: Remove aliases in //docs (#3125) Fixes #2976. The rest of it was fixed in #3045. --- docs/BUILD.bazel | 36 ---------------------------------- sphinxdocs/private/BUILD.bazel | 2 +- 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index 852c4d4fa6..c1009b7313 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@dev_pip//:requirements.bzl", "requirement") load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED") # buildifier: disable=bzl-visibility @@ -189,38 +188,3 @@ lock( ], visibility = ["//:__subpackages__"], ) - -# Temporary compatibility aliases for some other projects depending on the old -# bzl_library targets. -alias( - name = "defs", - actual = "//python:defs_bzl", - deprecation = "Use //python:defs_bzl instead; targets under //docs are internal.", - visibility = ["//visibility:public"], -) - -alias( - name = "bazel_repo_tools", - actual = "//python/private:bazel_tools_bzl", - deprecation = "Use @bazel_tools//tools:bzl_srcs instead; targets under //docs are internal.", - visibility = ["//visibility:public"], -) - -bzl_library( - name = "pip_install_bzl", - deprecation = "Use //python:pip_bzl or //python/pip_install:pip_repository_bzl instead; " + - "targets under //docs are internal.", - visibility = ["//visibility:public"], - deps = [ - "//python:pip_bzl", - "//python/pip_install:pip_repository_bzl", - ], -) - -alias( - name = "requirements_parser_bzl", - actual = "//python/pip_install:pip_repository_bzl", - deprecation = "Use //python/pip_install:pip_repository_bzl instead; Both the requirements " + - "parser and targets under //docs are internal", - visibility = ["//visibility:public"], -) diff --git a/sphinxdocs/private/BUILD.bazel b/sphinxdocs/private/BUILD.bazel index c4246ed0de..c707b4d1d8 100644 --- a/sphinxdocs/private/BUILD.bazel +++ b/sphinxdocs/private/BUILD.bazel @@ -13,7 +13,7 @@ # limitations under the License. load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("//python:proto.bzl", "py_proto_library") +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") load("//python:py_binary.bzl", "py_binary") load("//python:py_library.bzl", "py_library") From 9889d9f6b06fcd058ccbbea856fa1c90f3bbc216 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Fri, 1 Aug 2025 16:16:43 -0700 Subject: [PATCH 19/28] fix(gazelle): Rename experimental_allow_relative_imports directive to follow convention (#3128) Prefix `experimental_allow_relative_imports` with `python_` to match the rest of the directives. 1.6.0 hasn't been released yet, so this is a non-breaking change. --- CHANGELOG.md | 2 +- gazelle/README.md | 4 ++-- .../python/testdata/relative_imports_package_mode/BUILD.in | 2 +- .../python/testdata/relative_imports_package_mode/BUILD.out | 2 +- gazelle/python/testdata/sibling_imports_disabled/BUILD.in | 2 +- gazelle/python/testdata/sibling_imports_disabled/BUILD.out | 2 +- gazelle/python/testdata/sibling_imports_disabled/README.md | 2 +- .../testdata/sibling_imports_disabled_file_mode/BUILD.in | 2 +- .../testdata/sibling_imports_disabled_file_mode/BUILD.out | 2 +- .../testdata/sibling_imports_disabled_file_mode/README.md | 2 +- gazelle/pythonconfig/pythonconfig.go | 2 +- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d21ebc7d36..f69e94ec65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,7 +56,7 @@ END_UNRELEASED_TEMPLATE ### Changed * (gazelle) For package mode, resolve dependencies when imports are relative to the package path. This is enabled via the - `# gazelle:experimental_allow_relative_imports` true directive ({gh-issue}`2203`). + `# gazelle:python_experimental_allow_relative_imports` true directive ({gh-issue}`2203`). * (gazelle) Types for exposed members of `python.ParserOutput` are now all public. * (gazelle) Removed the requirement for `__init__.py`, `__main__.py`, or `__test__.py` files to be present in a directory to generate a `BUILD.bazel` file. diff --git a/gazelle/README.md b/gazelle/README.md index 8b088a4e70..83f341c49d 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -222,7 +222,7 @@ Python-specific directives are as follows: | Defines the format of the distribution name in labels to third-party deps. Useful for using Gazelle plugin with other rules with different repository conventions (e.g. `rules_pycross`). Full label is always prepended with (pip) repository name, e.g. `@pip//numpy`. | | `# gazelle:python_label_normalization` | `snake_case` | | Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503". | -| `# gazelle:experimental_allow_relative_imports` | `false` | +| `# gazelle:python_experimental_allow_relative_imports` | `false` | | Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".| | `# gazelle:python_generate_pyi_deps` | `false` | | Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. | @@ -736,7 +736,7 @@ See [Issue #3076][gh3076] for more information. [gh3076]: https://github.com/bazel-contrib/rules_python/issues/3076 -#### Directive: `experimental_allow_relative_imports` +#### Directive: `python_experimental_allow_relative_imports` Enables experimental support for resolving relative imports in `python_generation_mode package`. diff --git a/gazelle/python/testdata/relative_imports_package_mode/BUILD.in b/gazelle/python/testdata/relative_imports_package_mode/BUILD.in index 78ef0a7863..52bcb68600 100644 --- a/gazelle/python/testdata/relative_imports_package_mode/BUILD.in +++ b/gazelle/python/testdata/relative_imports_package_mode/BUILD.in @@ -1,2 +1,2 @@ # gazelle:python_generation_mode package -# gazelle:experimental_allow_relative_imports true +# gazelle:python_experimental_allow_relative_imports true diff --git a/gazelle/python/testdata/relative_imports_package_mode/BUILD.out b/gazelle/python/testdata/relative_imports_package_mode/BUILD.out index f51b516cab..8775c114ef 100644 --- a/gazelle/python/testdata/relative_imports_package_mode/BUILD.out +++ b/gazelle/python/testdata/relative_imports_package_mode/BUILD.out @@ -1,7 +1,7 @@ load("@rules_python//python:defs.bzl", "py_binary") # gazelle:python_generation_mode package -# gazelle:experimental_allow_relative_imports true +# gazelle:python_experimental_allow_relative_imports true py_binary( name = "relative_imports_package_mode_bin", diff --git a/gazelle/python/testdata/sibling_imports_disabled/BUILD.in b/gazelle/python/testdata/sibling_imports_disabled/BUILD.in index 9509fd9727..44f7406e58 100644 --- a/gazelle/python/testdata/sibling_imports_disabled/BUILD.in +++ b/gazelle/python/testdata/sibling_imports_disabled/BUILD.in @@ -1,2 +1,2 @@ # gazelle:python_resolve_sibling_imports false -# gazelle:experimental_allow_relative_imports true +# gazelle:python_experimental_allow_relative_imports true diff --git a/gazelle/python/testdata/sibling_imports_disabled/BUILD.out b/gazelle/python/testdata/sibling_imports_disabled/BUILD.out index 7568f38f50..d3d5c6bfab 100644 --- a/gazelle/python/testdata/sibling_imports_disabled/BUILD.out +++ b/gazelle/python/testdata/sibling_imports_disabled/BUILD.out @@ -1,7 +1,7 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") # gazelle:python_resolve_sibling_imports false -# gazelle:experimental_allow_relative_imports true +# gazelle:python_experimental_allow_relative_imports true py_library( name = "sibling_imports_disabled", diff --git a/gazelle/python/testdata/sibling_imports_disabled/README.md b/gazelle/python/testdata/sibling_imports_disabled/README.md index d534a44bf1..a39023e8a3 100644 --- a/gazelle/python/testdata/sibling_imports_disabled/README.md +++ b/gazelle/python/testdata/sibling_imports_disabled/README.md @@ -10,7 +10,7 @@ disabled. It covers different types of imports in `pkg/unit_test.py`: (not the sibling `typing.py`). - `from .b import run` / `from .typing import A` - resolves to the sibling `pkg/b.py` / `pkg/typing.py` (with - `gazelle:experimental_allow_relative_imports` enabled) + `gazelle:python_experimental_allow_relative_imports` enabled) - `import test_util` - resolves to the root-level `test_util.py` instead of the sibling `pkg/test_util.py` - `from b import run` - resolves to the root-level `b.py` instead of the diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in index 04494394c7..32b0bec20f 100644 --- a/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.in @@ -1,3 +1,3 @@ # gazelle:python_generation_mode file # gazelle:python_resolve_sibling_imports false -# gazelle:experimental_allow_relative_imports true +# gazelle:python_experimental_allow_relative_imports true diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out index da53e14864..d7a829e8ea 100644 --- a/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/BUILD.out @@ -2,7 +2,7 @@ load("@rules_python//python:defs.bzl", "py_library", "py_test") # gazelle:python_generation_mode file # gazelle:python_resolve_sibling_imports false -# gazelle:experimental_allow_relative_imports true +# gazelle:python_experimental_allow_relative_imports true py_library( name = "a", diff --git a/gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md b/gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md index 0bfbcffb58..124e751b10 100644 --- a/gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md +++ b/gazelle/python/testdata/sibling_imports_disabled_file_mode/README.md @@ -10,7 +10,7 @@ disabled. It covers different types of imports in `pkg/unit_test.py`: (not the sibling `typing.py`). - `from .b import run` / `from .typing import A` - resolves to the sibling `pkg/b.py` / `pkg/typing.py` (with - `gazelle:experimental_allow_relative_imports` enabled) + `gazelle:python_experimental_allow_relative_imports` enabled) - `import test_util` - resolves to the root-level `test_util.py` instead of the sibling `pkg/test_util.py` - `from b import run` - resolves to the root-level `b.py` instead of the diff --git a/gazelle/pythonconfig/pythonconfig.go b/gazelle/pythonconfig/pythonconfig.go index b3d56591ee..ed9b914e82 100644 --- a/gazelle/pythonconfig/pythonconfig.go +++ b/gazelle/pythonconfig/pythonconfig.go @@ -99,7 +99,7 @@ const ( LabelNormalization = "python_label_normalization" // ExperimentalAllowRelativeImports represents the directive that controls // whether relative imports are allowed. - ExperimentalAllowRelativeImports = "experimental_allow_relative_imports" + ExperimentalAllowRelativeImports = "python_experimental_allow_relative_imports" // GeneratePyiDeps represents the directive that controls whether to generate // separate pyi_deps attribute or merge type-checking dependencies into deps. // Defaults to false for backward compatibility. From 6819b844e6fa80c18ebd61090b1c9b0f7abf623b Mon Sep 17 00:00:00 2001 From: Philipp Schrader Date: Fri, 1 Aug 2025 16:19:56 -0700 Subject: [PATCH 20/28] test: Print REPL error message during test failures (#3124) I noticed that the CI error messages for #3114 are not useful. This patch aims to help with that by printing the output of the REPL code. If there's an exception during startup for example, then the test log will now contain the stack trace. --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- tests/repl/repl_test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/repl/repl_test.py b/tests/repl/repl_test.py index 37c9a37a0d..01d0442922 100644 --- a/tests/repl/repl_test.py +++ b/tests/repl/repl_test.py @@ -29,13 +29,16 @@ def setUp(self): def run_code_in_repl(self, lines: Iterable[str], *, env=None) -> str: """Runs the lines of code in the REPL and returns the text output.""" - return subprocess.check_output( - [self.repl], - text=True, - stderr=subprocess.STDOUT, - input="\n".join(lines), - env=env, - ).strip() + try: + return subprocess.check_output( + [self.repl], + text=True, + stderr=subprocess.STDOUT, + input="\n".join(lines), + env=env, + ).strip() + except subprocess.CalledProcessError as error: + raise RuntimeError(f"Failed to run the REPL:\n{error.stdout}") from error def test_repl_version(self): """Validates that we can successfully execute arbitrary code on the REPL.""" From 39703fa18135fae3bcc458490a0b8266ca85e513 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Fri, 1 Aug 2025 21:58:28 -0700 Subject: [PATCH 21/28] docs(gazelle): Start migrating Gazelle docs to ReadTheDocs, part 1 of ~5 (#3129) Part of #3082 First of probably 5 PRs. + Set up the Bazel config so that the rules_python root docs can reference and use the gazelle/docs directory. + Replace the original `docs/gazelle.md` with `gazelle/docs/index.md` + Migrate general info from `gazelle/README.md` to `gazelle/docs/index.md` + Mechanical updates: + Wrap at ~80 chars + Use MyST directives and roles. + Also a drive-by update to building and running docs _without_ `ibazel`. --- docs/BUILD.bazel | 2 +- docs/README.md | 14 +++++++++++++ docs/gazelle.md | 9 -------- docs/index.md | 2 +- gazelle/README.md | 29 ++++---------------------- gazelle/docs/BUILD.bazel | 5 +++++ gazelle/docs/index.md | 45 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 70 insertions(+), 36 deletions(-) delete mode 100644 docs/gazelle.md create mode 100644 gazelle/docs/BUILD.bazel create mode 100644 gazelle/docs/index.md diff --git a/docs/BUILD.bazel b/docs/BUILD.bazel index c1009b7313..fdb74f9407 100644 --- a/docs/BUILD.bazel +++ b/docs/BUILD.bazel @@ -55,7 +55,7 @@ sphinx_docs( "_*", "*.inv*", ], - ), + ) + ["//gazelle/docs"], config = "conf.py", formats = [ "html", diff --git a/docs/README.md b/docs/README.md index 456f1cfd64..1316d733bb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,20 @@ changes and re-run the build process, and you can simply refresh your browser to see the changes. Using ibazel is not required; you can manually run the equivalent bazel command if desired. +An alternative to `ibazel` is using `inotify` on Linux systems: + +``` +inotifywait --event modify --monitor . --recursive --includei '^.*\.md$' | +while read -r dir events filename; do bazel build //docs:docs; done; +``` + +And lastly, a poor-man's `ibazel` and `inotify` is simply `watch` with +a reasonable interval like 10s: + +``` +watch --interval 10 bazel build //docs:docs +``` + ### Installing ibazel The `ibazel` tool can be used to automatically rebuild the docs as you diff --git a/docs/gazelle.md b/docs/gazelle.md deleted file mode 100644 index 60b46faf2c..0000000000 --- a/docs/gazelle.md +++ /dev/null @@ -1,9 +0,0 @@ -# Gazelle plugin - -[Gazelle](https://github.com/bazelbuild/bazel-gazelle) -is a build file generator for Bazel projects. It can create new `BUILD.bazel` files for a project that follows language conventions and update existing build files to include new sources, dependencies, and options. - -Bazel may run Gazelle using the Gazelle rule, or Gazelle may be installed and run as a command line tool. - -See the documentation for Gazelle with `rules_python` in the {gh-path}`gazelle` -directory. diff --git a/docs/index.md b/docs/index.md index 25b423c6c3..bdc6982ad5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -99,7 +99,7 @@ pypi/index Toolchains coverage precompiling -gazelle +gazelle/docs/index REPL Extending Contributing diff --git a/gazelle/README.md b/gazelle/README.md index 83f341c49d..df3085bb37 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -1,19 +1,10 @@ # Python Gazelle plugin -[Gazelle](https://github.com/bazelbuild/bazel-gazelle) -is a build file generator for Bazel projects. It can create new BUILD.bazel files for a project that follows language conventions, and it can update existing build files to include new sources, dependencies, and options. +:::{note} +The gazelle plugin docs are being migrated to our primary documentation on +ReadTheDocs. Please see https://rules-python.readthedocs.io/gazelle/docs/index.html. +::: -Gazelle may be run by Bazel using the gazelle rule, or it may be installed and run as a command line tool. - -This directory contains a plugin for -[Gazelle](https://github.com/bazelbuild/bazel-gazelle) -that generates BUILD files content for Python code. When Gazelle is run as a command line tool with this plugin, it embeds a Python interpreter resolved during the plugin build. -The behavior of the plugin is slightly different with different version of the interpreter as the Python `stdlib` changes with every minor version release. -Distributors of Gazelle binaries should, therefore, build a Gazelle binary for each OS+CPU architecture+Minor Python version combination they are targeting. - -The following instructions are for when you use [bzlmod](https://docs.bazel.build/versions/5.0.0/bzlmod.html). -Please refer to older documentation that includes instructions on how to use Gazelle -without using bzlmod as your dependency manager. ## Example @@ -153,18 +144,6 @@ gazelle( That's it, now you can finally run `bazel run //:gazelle` anytime you edit Python code, and it should update your `BUILD` files correctly. -## Usage - -Gazelle is non-destructive. -It will try to leave your edits to BUILD files alone, only making updates to `py_*` targets. -However it will remove dependencies that appear to be unused, so it's a -good idea to check in your work before running Gazelle so you can easily -revert any changes it made. - -The rules_python extension assumes some conventions about your Python code. -These are noted below, and might require changes to your existing code. - -Note that the `gazelle` program has multiple commands. At present, only the `update` command (the default) does anything for Python code. ### Directives diff --git a/gazelle/docs/BUILD.bazel b/gazelle/docs/BUILD.bazel new file mode 100644 index 0000000000..7c6b6fd56e --- /dev/null +++ b/gazelle/docs/BUILD.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "docs", + srcs = glob(["*.md"]), + visibility = ["//visibility:public"], +) diff --git a/gazelle/docs/index.md b/gazelle/docs/index.md new file mode 100644 index 0000000000..ea20e9c3e0 --- /dev/null +++ b/gazelle/docs/index.md @@ -0,0 +1,45 @@ +# Gazelle Plugin + +[Gazelle][gazelle] is a build file generator for Bazel projects. It can +create new `BUILD` or `BUILD.bazel` files for a project that +follows language conventions and update existing build files to include new +sources, dependencies, and options. + +[gazelle]: https://github.com/bazel-contrib/bazel-gazelle + +Bazel may run Gazelle using the Gazelle rule, or Gazelle may be installed and run +as a command line tool. + +The {gh-path}`gazelle` directory contains a plugin for Gazelle +that generates `BUILD` files content for Python code. When Gazelle is +run as a command line tool with this plugin, it embeds a Python interpreter +resolved during the plugin build. The behavior of the plugin is slightly +different with different version of the interpreter as the Python +`stdlib` changes with every minor version release. Distributors of Gazelle +binaries should, therefore, build a Gazelle binary for each OS+CPU +architecture+Minor Python version combination they are targeting. + +:::{note} +These instructions are for when you use [bzlmod][bzlmod]. Please refer to +older documentation that includes instructions on how to use Gazelle +without using bzlmod as your dependency manager. +::: + +[bzlmod]: https://bazel.build/external/module + +Gazelle is non-destructive. It will try to leave your edits to `BUILD` +files alone, only making updates to `py_*` targets. However it **will +remove** dependencies that appear to be unused, so it's a good idea to check +in your work before running Gazelle so you can easily revert any changes it made. + +The `rules_python` extension assumes some conventions about your Python code. +These are noted in the subsequent documents, and might require changes to your +existing code. + +Note that the `gazelle` program has multiple commands. At present, only +the `update` command (the default) does anything for Python code. + + +```{toctree} +:maxdepth: 1 +``` From 2fc5f1cc5635be08dbebd0e0c448afb67c3f251e Mon Sep 17 00:00:00 2001 From: Jonathan Woodbury Date: Sun, 3 Aug 2025 03:14:51 -0400 Subject: [PATCH 22/28] feat(repl): add tab completion on platforms with readline support (#3114) This adds tab completion to the default stub when using the REPL feature. However, the feature only works in environments with `readline` support, which means that with the bundled toolchains, Windows will not have tab completion. Work towards #3090 --------- Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com> --- CHANGELOG.md | 3 +++ python/bin/repl_stub.py | 44 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f69e94ec65..422e399026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -104,6 +104,9 @@ END_UNRELEASED_TEMPLATE {#v0-0-0-added} ### Added +* (repl) Default stub now has tab completion, where `readline` support is available, + see ([#3114](https://github.com/bazel-contrib/rules_python/pull/3114)). + ([#3114](https://github.com/bazel-contrib/rules_python/pull/3114)). * (pypi) To configure the environment for `requirements.txt` evaluation, use the newly added developer preview of the `pip.default` tag class. Only `rules_python` and root modules can use this feature. You can also configure custom `config_settings` using `pip.default`. diff --git a/python/bin/repl_stub.py b/python/bin/repl_stub.py index 1e21b26dc3..f5b7c0aa4f 100644 --- a/python/bin/repl_stub.py +++ b/python/bin/repl_stub.py @@ -17,8 +17,28 @@ console_locals = globals().copy() import code +import rlcompleter import sys + +class DynamicCompleter(rlcompleter.Completer): + """ + A custom completer that dynamically updates its namespace to include new + imports made within the interactive session. + """ + + def __init__(self, namespace): + # Store a reference to the namespace, not a copy, so that changes to the namespace are + # reflected. + self.namespace = namespace + + def complete(self, text, state): + # Update the completer's internal namespace with the current interactive session's locals + # and globals. This is the key to making new imports discoverable. + rlcompleter.Completer.__init__(self, self.namespace) + return super().complete(text, state) + + if sys.stdin.isatty(): # Use the default options. exitmsg = None @@ -28,5 +48,29 @@ sys.ps1 = "" sys.ps2 = "" +# Set up tab completion. +try: + import readline + + completer = DynamicCompleter(console_locals) + readline.set_completer(completer.complete) + + # TODO(jpwoodbu): Use readline.backend instead of readline.__doc__ once we can depend on having + # Python >=3.13. + if "libedit" in readline.__doc__: # type: ignore + readline.parse_and_bind("bind ^I rl_complete") + elif "GNU readline" in readline.__doc__: # type: ignore + readline.parse_and_bind("tab: complete") + else: + print( + "Could not enable tab completion: " + "unable to determine readline backend" + ) +except ImportError: + print( + "Could not enable tab completion: " + "readline module not available on this platform" + ) + # We set the banner to an empty string because the repl_template.py file already prints the banner. code.interact(local=console_locals, banner="", exitmsg=exitmsg) From 54397d71031de37fbc810922244e0cc4788713fd Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Sun, 3 Aug 2025 21:11:20 -0700 Subject: [PATCH 23/28] docs(gazelle): Migrate Gazelle docs to ReadTheDocs, part 2/5: installation and usage (#3132) Part of #3082 2nd of probably 5 PRs. + Migrate installation and usage info from `gazelle/README.md` to `gazelle/docs/installation_and_usage.md` + Slight rewording and reformatting of the `Example` section. + Reorganized and modernized the `MODULE.bazel` and `BUILD.bazel` examples: + less details on rules_python as that's available in other docs + default to gazelle multilang support now that #3057 is merged. + Mechanical updates: + Wrap at ~80 chars + Use MyST directives and roles. --- gazelle/README.md | 139 ----------------------- gazelle/docs/index.md | 1 + gazelle/docs/installation_and_usage.md | 151 +++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 139 deletions(-) create mode 100644 gazelle/docs/installation_and_usage.md diff --git a/gazelle/README.md b/gazelle/README.md index df3085bb37..11a9e5b2ba 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -6,145 +6,6 @@ ReadTheDocs. Please see https://rules-python.readthedocs.io/gazelle/docs/index.h ::: -## Example - -We have an example of using Gazelle with Python located [here](https://github.com/bazel-contrib/rules_python/tree/main/examples/bzlmod). -A fully-working example without using bzlmod is in [`examples/build_file_generation`](../examples/build_file_generation). - -The following documentation covers using bzlmod. - -## Adding Gazelle to your project - -First, you'll need to add Gazelle to your `MODULE.bazel` file. -Get the current version of Gazelle from there releases here: https://github.com/bazelbuild/bazel-gazelle/releases/. - - -See the installation `MODULE.bazel` snippet on the Releases page: -https://github.com/bazel-contrib/rules_python/releases in order to configure rules_python. - -You will also need to add the `bazel_dep` for configuration for `rules_python_gazelle_plugin`. - -Here is a snippet of a `MODULE.bazel` file. - -```starlark -# The following stanza defines the dependency rules_python. -bazel_dep(name = "rules_python", version = "0.22.0") - -# The following stanza defines the dependency rules_python_gazelle_plugin. -# For typical setups you set the version. -bazel_dep(name = "rules_python_gazelle_plugin", version = "0.22.0") - -# The following stanza defines the dependency gazelle. -bazel_dep(name = "gazelle", version = "0.31.0", repo_name = "bazel_gazelle") - -# Import the python repositories generated by the given module extension into the scope of the current module. -use_repo(python, "python3_9") -use_repo(python, "python3_9_toolchains") - -# Register an already-defined toolchain so that Bazel can use it during toolchain resolution. -register_toolchains( - "@python3_9_toolchains//:all", -) - -# Use the pip extension -pip = use_extension("@rules_python//python:extensions.bzl", "pip") - -# Use the extension to call the `pip_repository` rule that invokes `pip`, with `incremental` set. -# Accepts a locked/compiled requirements file and installs the dependencies listed within. -# Those dependencies become available in a generated `requirements.bzl` file. -# You can instead check this `requirements.bzl` file into your repo. -# Because this project has different requirements for windows vs other -# operating systems, we have requirements for each. -pip.parse( - name = "pip", - requirements_lock = "//:requirements_lock.txt", - requirements_windows = "//:requirements_windows.txt", -) - -# Imports the pip toolchain generated by the given module extension into the scope of the current module. -use_repo(pip, "pip") -``` -Next, we'll fetch metadata about your Python dependencies, so that gazelle can -determine which package a given import statement comes from. This is provided -by the `modules_mapping` rule. We'll make a target for consuming this -`modules_mapping`, and writing it as a manifest file for Gazelle to read. -This is checked into the repo for speed, as it takes some time to calculate -in a large monorepo. - -Gazelle will walk up the filesystem from a Python file to find this metadata, -looking for a file called `gazelle_python.yaml` in an ancestor folder of the Python code. -Create an empty file with this name. It might be next to your `requirements.txt` file. -(You can just use `touch` at this point, it just needs to exist.) - -To keep the metadata updated, put this in your `BUILD.bazel` file next to `gazelle_python.yaml`: - -```starlark -load("@pip//:requirements.bzl", "all_whl_requirements") -load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest") -load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") - -# This rule fetches the metadata for python packages we depend on. That data is -# required for the gazelle_python_manifest rule to update our manifest file. -modules_mapping( - name = "modules_map", - wheels = all_whl_requirements, -) - -# Gazelle python extension needs a manifest file mapping from -# an import to the installed package that provides it. -# This macro produces two targets: -# - //:gazelle_python_manifest.update can be used with `bazel run` -# to recalculate the manifest -# - //:gazelle_python_manifest.test is a test target ensuring that -# the manifest doesn't need to be updated -gazelle_python_manifest( - name = "gazelle_python_manifest", - modules_mapping = ":modules_map", - # This is what we called our `pip_parse` rule, where third-party - # python libraries are loaded in BUILD files. - pip_repository_name = "pip", - # This should point to wherever we declare our python dependencies - # (the same as what we passed to the modules_mapping rule in WORKSPACE) - # This argument is optional. If provided, the `.test` target is very - # fast because it just has to check an integrity field. If not provided, - # the integrity field is not added to the manifest which can help avoid - # merge conflicts in large repos. - requirements = "//:requirements_lock.txt", - # include_stub_packages: bool (default: False) - # If set to True, this flag automatically includes any corresponding type stub packages - # for the third-party libraries that are present and used. For example, if you have - # `boto3` as a dependency, and this flag is enabled, the corresponding `boto3-stubs` - # package will be automatically included in the BUILD file. - # - # Enabling this feature helps ensure that type hints and stubs are readily available - # for tools like type checkers and IDEs, improving the development experience and - # reducing manual overhead in managing separate stub packages. - include_stub_packages = True -) -``` - -Finally, you create a target that you'll invoke to run the Gazelle tool -with the rules_python extension included. This typically goes in your root -`/BUILD.bazel` file: - -```starlark -load("@bazel_gazelle//:def.bzl", "gazelle") - -# Our gazelle target points to the python gazelle binary. -# This is the simple case where we only need one language supported. -# If you also had proto, go, or other gazelle-supported languages, -# you would also need a gazelle_binary rule. -# See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example -gazelle( - name = "gazelle", - gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary", -) -``` - -That's it, now you can finally run `bazel run //:gazelle` anytime -you edit Python code, and it should update your `BUILD` files correctly. - - ### Directives You can configure the extension using directives, just like for other diff --git a/gazelle/docs/index.md b/gazelle/docs/index.md index ea20e9c3e0..c04efd6a41 100644 --- a/gazelle/docs/index.md +++ b/gazelle/docs/index.md @@ -42,4 +42,5 @@ the `update` command (the default) does anything for Python code. ```{toctree} :maxdepth: 1 +installation_and_usage ``` diff --git a/gazelle/docs/installation_and_usage.md b/gazelle/docs/installation_and_usage.md new file mode 100644 index 0000000000..e764957581 --- /dev/null +++ b/gazelle/docs/installation_and_usage.md @@ -0,0 +1,151 @@ +# Installation and Usage + +## Example + +Examples of using Gazelle with Python can be found in the `rules_python` +repo: + +* bzlmod: {gh-path}`examples/bzlmod_build_file_generation` +* WORKSPACE: {gh-path}`examples/build_file_generation` + +:::{note} +The following documentation covers using bzlmod. +::: + + +## Adding Gazelle to your project + +First, you'll need to add Gazelle to your `MODULE.bazel` file. Get the current +version of [Gazelle][bcr-gazelle] from the [Bazel Central Registry][bcr]. Then +do the same for [`rules_python`][bcr-rules-python] and +[`rules_python_gazelle_plugin`][bcr-rules-python-gazelle-plugin]. + +[bcr-gazelle]: https://registry.bazel.build/modules/gazelle +[bcr]: https://registry.bazel.build/ +[bcr-rules-python]: https://registry.bazel.build/modules/rules_python +[bcr-rules-python-gazelle-plugin]: https://registry.bazel.build/modules/rules_python_gazelle_plugin + +Here is a snippet of a `MODULE.bazel` file. Note that most of it is just +general config for `rules_python` itself - the Gazelle plugin is only two lines +at the end. + +```starlark +################################################ +## START rules_python CONFIG ## +## See the main rules_python docs for details ## +################################################ +bazel_dep(name = "rules_python", version = "1.5.1") + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain(python_version = "3.12.2") +use_repo(python, "python_3_12_2") + +pip = use_extension("@rules_python//python:extensions.bzl", "pip") +pip.parse( + hub_name = "pip", + requirements_lock = "//:requirements_lock.txt", + requirements_windows = "//:requirements_windows.txt", +) +use_repo(pip, "pip") + +############################################## +## START rules_python_gazelle_plugin CONFIG ## +############################################## + +# The Gazelle plugin depends on Gazelle. +bazel_dep(name = "gazelle", version = "0.33.0", repo_name = "bazel_gazelle") + +# Typically rules_python_gazelle_plugin is version matched to rules_python. +bazel_dep(name = "rules_python_gazelle_plugin", version = "1.5.1") +``` + +Next, we'll fetch metadata about your Python dependencies, so that gazelle can +determine which package a given import statement comes from. This is provided +by the `modules_mapping` rule. We'll make a target for consuming this +`modules_mapping`, and writing it as a manifest file for Gazelle to read. +This is checked into the repo for speed, as it takes some time to calculate +in a large monorepo. + +Gazelle will walk up the filesystem from a Python file to find this metadata, +looking for a file called `gazelle_python.yaml` in an ancestor folder +of the Python code. Create an empty file with this name. It might be next +to your `requirements.txt` file. (You can just use {command}`touch` at +this point, it just needs to exist.) + +To keep the metadata updated, put this in your `BUILD.bazel` file next +to `gazelle_python.yaml`: + +```starlark +# `@pip` is the hub_name from pip.parse in MODULE.bazel. +load("@pip//:requirements.bzl", "all_whl_requirements") +load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest") +load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") + +# This rule fetches the metadata for python packages we depend on. That data is +# required for the gazelle_python_manifest rule to update our manifest file. +modules_mapping( + name = "modules_map", + wheels = all_whl_requirements, +) + +# Gazelle python extension needs a manifest file mapping from +# an import to the installed package that provides it. +# This macro produces two targets: +# - //:gazelle_python_manifest.update can be used with `bazel run` +# to recalculate the manifest +# - //:gazelle_python_manifest.test is a test target ensuring that +# the manifest doesn't need to be updated +gazelle_python_manifest( + name = "gazelle_python_manifest", + modules_mapping = ":modules_map", + + # This is what we called our `pip.parse` rule in MODULE.bazel, where third-party + # python libraries are loaded in BUILD files. + pip_repository_name = "pip", + + # This should point to wherever we declare our python dependencies + # (the same as what we passed to the modules_mapping rule in WORKSPACE) + # This argument is optional. If provided, the `.test` target is very + # fast because it just has to check an integrity field. If not provided, + # the integrity field is not added to the manifest which can help avoid + # merge conflicts in large repos. + requirements = "//:requirements_lock.txt", + + # include_stub_packages: bool (default: False) + # If set to True, this flag automatically includes any corresponding type stub packages + # for the third-party libraries that are present and used. For example, if you have + # `boto3` as a dependency, and this flag is enabled, the corresponding `boto3-stubs` + # package will be automatically included in the BUILD file. + # Enabling this feature helps ensure that type hints and stubs are readily available + # for tools like type checkers and IDEs, improving the development experience and + # reducing manual overhead in managing separate stub packages. + include_stub_packages = True +) +``` + +Finally, you create a target that you'll invoke to run the Gazelle tool +with the `rules_python` extension included. This typically goes in your root +`/BUILD.bazel` file: + +```starlark +load("@bazel_gazelle//:def.bzl", "gazelle", "gazelle_binary") + +gazelle_binary( + name = "gazelle_multilang", + languages = [ + # List of language plugins. + # If you want to generate py_proto_library targets PR #3057), then + # the proto language plugin _must_ come before the rules_python plugin. + #"@bazel_gazelle//lanugage/proto", + "@rules_python_gazelle_plugin//python", + ], +) + +gazelle( + name = "gazelle", + gazelle = ":gazelle_multilang", +) +``` + +That's it, now you can finally run `bazel run //:gazelle` anytime +you edit Python code, and it should update your `BUILD` files correctly. From 3c88a5bec26351da1daa1a331f094f11e68e7664 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Mon, 4 Aug 2025 09:59:57 -0700 Subject: [PATCH 24/28] docs(gazelle): Migrate Gazelle docs to ReadTheDocs, part 3/5: annotations (#3137) Part of #3082 3rd of probably 5 PRs. + Migrate annotations docs from `gazelle/README.md` to `gazelle/docs/annotations.md` + Switch from table-based summary to bulleted lists + This will be much easier to maintain going forward. + Mechanical updates: + Wrap at ~80 chars + Use MyST directives and roles. --- gazelle/README.md | 185 ---------------------------------- gazelle/docs/annotations.md | 194 ++++++++++++++++++++++++++++++++++++ gazelle/docs/index.md | 1 + 3 files changed, 195 insertions(+), 185 deletions(-) create mode 100644 gazelle/docs/annotations.md diff --git a/gazelle/README.md b/gazelle/README.md index 11a9e5b2ba..efc7004eaf 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -390,191 +390,6 @@ py_proto_library( When `false`, Gazelle will ignore any `py_proto_library`, including previously-generated or hand-created rules. -### Annotations - -*Annotations* refer to comments found _within Python files_ that configure how -Gazelle acts for that particular file. - -Annotations have the form: - -```python -# gazelle:annotation_name value -``` - -and can reside anywhere within a Python file where comments are valid. For example: - -```python -import foo -# gazelle:annotation_name value - -def bar(): # gazelle:annotation_name value - pass -``` - -The annotations are: - -| **Annotation** | **Default value** | -|---------------------------------------------------------------|-------------------| -| [`# gazelle:ignore imports`](#annotation-ignore) | N/A | -| Tells Gazelle to ignore import statements. `imports` is a comma-separated list of imports to ignore. | | -| [`# gazelle:include_dep targets`](#annotation-include_dep) | N/A | -| Tells Gazelle to include a set of dependencies, even if they are not imported in a Python module. `targets` is a comma-separated list of target names to include as dependencies. | | -| [`# gazelle:include_pytest_conftest bool`](#annotation-include_pytest_conftest) | N/A | -| Whether or not to include a sibling `:conftest` target in the deps of a `py_test` target. Default behaviour is to include `:conftest`. | | - - -#### Annotation: `ignore` - -This annotation accepts a comma-separated string of values. Values are names of Python -imports that Gazelle should _not_ include in target dependencies. - -The annotation can be added multiple times, and all values are combined and -de-duplicated. - -For `python_generation_mode = "package"`, the `ignore` annotations -found across all files included in the generated target are removed from `deps`. - -Example: - -```python -import numpy # a pypi package - -# gazelle:ignore bar.baz.hello,foo -import bar.baz.hello -import foo - -# Ignore this import because _reasons_ -import baz # gazelle:ignore baz -``` - -will cause Gazelle to generate: - -```starlark -deps = ["@pypi//numpy"], -``` - - -#### Annotation: `include_dep` - -This annotation accepts a comma-separated string of values. Values _must_ -be Python targets, but _no validation is done_. If a value is not a Python -target, building will result in an error saying: - -``` - does not have mandatory providers: 'PyInfo' or 'CcInfo' or 'PyInfo'. -``` - -Adding non-Python targets to the generated target is a feature request being -tracked in [Issue #1865](https://github.com/bazel-contrib/rules_python/issues/1865). - -The annotation can be added multiple times, and all values are combined -and de-duplicated. - -For `python_generation_mode = "package"`, the `include_dep` annotations -found across all files included in the generated target are included in `deps`. - -Example: - -```python -# gazelle:include_dep //foo:bar,:hello_world,//:abc -# gazelle:include_dep //:def,//foo:bar -import numpy # a pypi package -``` - -will cause Gazelle to generate: - -```starlark -deps = [ - ":hello_world", - "//:abc", - "//:def", - "//foo:bar", - "@pypi//numpy", -] -``` - -#### Annotation: `include_pytest_conftest` - -Added in [#3080][gh3080]. - -[gh3080]: https://github.com/bazel-contrib/rules_python/pull/3080 - -This annotation accepts any string that can be parsed by go's -[`strconv.ParseBool`][ParseBool]. If an unparsable string is passed, the -annotation is ignored. - -[ParseBool]: https://pkg.go.dev/strconv#ParseBool - -Starting with [`rules_python` 0.14.0][rules-python-0.14.0] (specifically [PR #879][gh879]), -Gazelle will include a `:conftest` dependency to an `py_test` target that is in -the same directory as `conftest.py`. - -[rules-python-0.14.0]: https://github.com/bazel-contrib/rules_python/releases/tag/0.14.0 -[gh879]: https://github.com/bazel-contrib/rules_python/pull/879 - -This annotation allows users to adjust that behavior. To disable the behavior, set -the annotation value to "false": - -``` -# some_file_test.py -# gazelle:include_pytest_conftest false -``` - -Example: - -Given a directory tree like: - -``` -. -├── BUILD.bazel -├── conftest.py -└── some_file_test.py -``` - -The default Gazelle behavior would create: - -```starlark -py_library( - name = "conftest", - testonly = True, - srcs = ["conftest.py"], - visibility = ["//:__subpackages__"], -) - -py_test( - name = "some_file_test", - srcs = ["some_file_test.py"], - deps = [":conftest"], -) -``` - -When `# gazelle:include_pytest_conftest false` is found in `some_file_test.py` - -```python -# some_file_test.py -# gazelle:include_pytest_conftest false -``` - -Gazelle will generate: - -```starlark -py_library( - name = "conftest", - testonly = True, - srcs = ["conftest.py"], - visibility = ["//:__subpackages__"], -) - -py_test( - name = "some_file_test", - srcs = ["some_file_test.py"], -) -``` - -See [Issue #3076][gh3076] for more information. - -[gh3076]: https://github.com/bazel-contrib/rules_python/issues/3076 - #### Directive: `python_experimental_allow_relative_imports` Enables experimental support for resolving relative imports in diff --git a/gazelle/docs/annotations.md b/gazelle/docs/annotations.md new file mode 100644 index 0000000000..cc87543c29 --- /dev/null +++ b/gazelle/docs/annotations.md @@ -0,0 +1,194 @@ +# Annotations + +*Annotations* refer to comments found _within Python files_ that configure how +Gazelle acts for that particular file. + +Annotations have the form: + +```python +# gazelle:annotation_name value +``` + +and can reside anywhere within a Python file where comments are valid. For example: + +```python +import foo +# gazelle:annotation_name value + +def bar(): # gazelle:annotation_name value + pass +``` + +The annotations are: + +* [`# gazelle:ignore imports`](#ignore) + * Default: n/a + * Allowed Values: A comma-separated string of python package names + * Tells Gazelle to ignore import statements. `imports` is a comma-separated + list of imports to ignore. +* [`# gazelle:include_dep targets`](#include-dep) + * Default: n/a + * Allowed Values: A string + * Tells Gazelle to include a set of dependencies, even if they are not imported + in a Python module. `targets` is a comma-separated list of target names + to include as dependencies. +* [`# gazelle:include_pytest_conftest bool`](#include-pytest-conftest) + * Default: n/a + * Allowed Values: `true`, `false` + * Whether or not to include a sibling `:conftest` target in the `deps` + of a {bzl:obj}`py_test` target. The default behaviour is to include `:conftest` + (i.e.: `# gazelle:include_pytest_conftest true`). + + +## `ignore` + +This annotation accepts a comma-separated string of values. Values are names of +Python imports that Gazelle should _not_ include in target dependencies. + +The annotation can be added multiple times, and all values are combined and +de-duplicated. + +For `python_generation_mode = "package"`, the `ignore` annotations +found across all files included in the generated target are removed from +`deps`. + +### Example: + +```python +import numpy # a pypi package + +# gazelle:ignore bar.baz.hello,foo +import bar.baz.hello +import foo + +# Ignore this import because _reasons_ +import baz # gazelle:ignore baz +``` + +will cause Gazelle to generate: + +```starlark +deps = ["@pypi//numpy"], +``` + + +## `include_dep` + +This annotation accepts a comma-separated string of values. Values _must_ +be Python targets, but _no validation is done_. If a value is not a Python +target, building will result in an error saying: + +``` + does not have mandatory providers: 'PyInfo' or 'CcInfo' or 'PyInfo'. +``` + +Adding non-Python targets to the generated target is a feature request being +tracked in {gh-issue}`1865`. + +The annotation can be added multiple times, and all values are combined +and de-duplicated. + +For `python_generation_mode = "package"`, the `include_dep` annotations +found across all files included in the generated target are included in +`deps`. + +### Example: + +```python +# gazelle:include_dep //foo:bar,:hello_world,//:abc +# gazelle:include_dep //:def,//foo:bar +import numpy # a pypi package +``` + +will cause Gazelle to generate: + +```starlark +deps = [ + ":hello_world", + "//:abc", + "//:def", + "//foo:bar", + "@pypi//numpy", +] +``` + + +## `include_pytest_conftest` + +:::{versionadded} VERSION_NEXT_FEATURE +{gh-pr}`3080` +::: + +This annotation accepts any string that can be parsed by go's +[`strconv.ParseBool`][ParseBool]. If an unparsable string is passed, the +annotation is ignored. + +[ParseBool]: https://pkg.go.dev/strconv#ParseBool + +Starting with [`rules_python` 0.14.0][rules-python-0.14.0] (specifically +{gh-pr}`879`), Gazelle will include a `:conftest` dependency to a +{bzl:obj}`py_test` target that is in the same directory as `conftest.py`. + +[rules-python-0.14.0]: https://github.com/bazel-contrib/rules_python/releases/tag/0.14.0 + +This annotation allows users to adjust that behavior. To disable the behavior, +set the annotation value to `false`: + +``` +# some_file_test.py +# gazelle:include_pytest_conftest false +``` + +### Example: + +Given a directory tree like: + +``` +. +├── BUILD.bazel +├── conftest.py +└── some_file_test.py +``` + +The default Gazelle behavior would create: + +```starlark +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "some_file_test", + srcs = ["some_file_test.py"], + deps = [":conftest"], +) +``` + +When `# gazelle:include_pytest_conftest false` is found in +`some_file_test.py` + +```python +# some_file_test.py +# gazelle:include_pytest_conftest false +``` + +Gazelle will generate: + +```starlark +py_library( + name = "conftest", + testonly = True, + srcs = ["conftest.py"], + visibility = ["//:__subpackages__"], +) + +py_test( + name = "some_file_test", + srcs = ["some_file_test.py"], +) +``` + +See {gh-issue}`3076` for more information. diff --git a/gazelle/docs/index.md b/gazelle/docs/index.md index c04efd6a41..e262d7ff46 100644 --- a/gazelle/docs/index.md +++ b/gazelle/docs/index.md @@ -43,4 +43,5 @@ the `update` command (the default) does anything for Python code. ```{toctree} :maxdepth: 1 installation_and_usage +annotations ``` From 768292440b242887f5488d378594838fe633360d Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Mon, 4 Aug 2025 22:20:55 -0700 Subject: [PATCH 25/28] docs(gazelle): Migrate Gazelle docs to ReadTheDocs, part 4/5: directives (#3139) Part of #3082 4th of probably 5 PRs. + Migrate directive docs from `gazelle/README.md` to `gazelle/docs/directives.md` + Switch from table-based summary to bulleted lists + This will be much easier to maintain going forward. + Add dedicated sections for each directive + Though not filled out yet. I do plan on filling them out later, I just can't say when. + Mechanical updates: + Wrap at ~80 chars + Use MyST directives and roles. --- gazelle/README.md | 424 ------------------------ gazelle/docs/directives.md | 647 +++++++++++++++++++++++++++++++++++++ gazelle/docs/index.md | 1 + 3 files changed, 648 insertions(+), 424 deletions(-) create mode 100644 gazelle/docs/directives.md diff --git a/gazelle/README.md b/gazelle/README.md index efc7004eaf..4de2c3c0cd 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -6,430 +6,6 @@ ReadTheDocs. Please see https://rules-python.readthedocs.io/gazelle/docs/index.h ::: -### Directives - -You can configure the extension using directives, just like for other -languages. These are just comments in the `BUILD.bazel` file which -govern behavior of the extension when processing files under that -folder. - -See https://github.com/bazelbuild/bazel-gazelle#directives -for some general directives that may be useful. -In particular, the `resolve` directive is language-specific -and can be used with Python. -Examples of these directives in use can be found in the -/gazelle/testdata folder in the rules_python repo. - -Python-specific directives are as follows: - -| **Directive** | **Default value** | -|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------| -| `# gazelle:python_extension` | `enabled` | -| Controls whether the Python extension is enabled or not. Sub-packages inherit this value. Can be either "enabled" or "disabled". | | -| [`# gazelle:python_root`](#directive-python_root) | n/a | -| Sets a Bazel package as a Python root. This is used on monorepos with multiple Python projects that don't share the top-level of the workspace as the root. See [Directive: `python_root`](#directive-python_root) below. | | -| `# gazelle:python_manifest_file_name` | `gazelle_python.yaml` | -| Overrides the default manifest file name. | | -| `# gazelle:python_ignore_files` | n/a | -| Controls the files which are ignored from the generated targets. | | -| `# gazelle:python_ignore_dependencies` | n/a | -| Controls the ignored dependencies from the generated targets. | | -| `# gazelle:python_validate_import_statements` | `true` | -| Controls whether the Python import statements should be validated. Can be "true" or "false" | | -| `# gazelle:python_generation_mode` | `package` | -| Controls the target generation mode. Can be "file", "package", or "project" | | -| `# gazelle:python_generation_mode_per_file_include_init` | `false` | -| Controls whether `__init__.py` files are included as srcs in each generated target when target generation mode is "file". Can be "true", or "false" | | -| [`# gazelle:python_generation_mode_per_package_require_test_entry_point`](#directive-python_generation_mode_per_package_require_test_entry_point) | `true` | -| Controls whether a file called `__test__.py` or a target called `__test__` is required to generate one test target per package in package mode. || -| `# gazelle:python_library_naming_convention` | `$package_name$` | -| Controls the `py_library` naming convention. It interpolates `$package_name$` with the Bazel package name. E.g. if the Bazel package name is `foo`, setting this to `$package_name$_my_lib` would result in a generated target named `foo_my_lib`. | | -| `# gazelle:python_binary_naming_convention` | `$package_name$_bin` | -| Controls the `py_binary` naming convention. Follows the same interpolation rules as `python_library_naming_convention`. | | -| `# gazelle:python_test_naming_convention` | `$package_name$_test` | -| Controls the `py_test` naming convention. Follows the same interpolation rules as `python_library_naming_convention`. | | -| [`# gazelle:python_proto_naming_convention`](#directive-python_proto_naming_convention) | `$proto_name$_py_pb2` | -| Controls the `py_proto_library` naming convention. It interpolates `$proto_name$` with the proto_library rule name, minus any trailing _proto. E.g. if the proto_library name is `foo_proto`, setting this to `$proto_name$_my_lib` would render to `foo_my_lib`. | | -| `# gazelle:resolve py ...` | n/a | -| Instructs the plugin what target to add as a dependency to satisfy a given import statement. The syntax is `# gazelle:resolve py import-string label` where `import-string` is the symbol in the python `import` statement, and `label` is the Bazel label that Gazelle should write in `deps`. | | -| [`# gazelle:python_default_visibility labels`](#directive-python_default_visibility) | | -| Instructs gazelle to use these visibility labels on all python targets. `labels` is a comma-separated list of labels (without spaces). | `//$python_root$:__subpackages__` | -| [`# gazelle:python_visibility label`](#directive-python_visibility) | | -| Appends additional visibility labels to each generated target. This directive can be set multiple times. | | -| [`# gazelle:python_test_file_pattern`](#directive-python_test_file_pattern) | `*_test.py,test_*.py` | -| Filenames matching these comma-separated `glob`s will be mapped to `py_test` targets. | -| `# gazelle:python_label_convention` | `$distribution_name$` | -| Defines the format of the distribution name in labels to third-party deps. Useful for using Gazelle plugin with other rules with different repository conventions (e.g. `rules_pycross`). Full label is always prepended with (pip) repository name, e.g. `@pip//numpy`. | -| `# gazelle:python_label_normalization` | `snake_case` | -| Controls how distribution names in labels to third-party deps are normalized. Useful for using Gazelle plugin with other rules with different label conventions (e.g. `rules_pycross` uses PEP-503). Can be "snake_case", "none", or "pep503". | -| `# gazelle:python_experimental_allow_relative_imports` | `false` | -| Controls whether Gazelle resolves dependencies for import statements that use paths relative to the current package. Can be "true" or "false".| -| `# gazelle:python_generate_pyi_deps` | `false` | -| Controls whether to generate a separate `pyi_deps` attribute for type-checking dependencies or merge them into the regular `deps` attribute. When `false` (default), type-checking dependencies are merged into `deps` for backward compatibility. When `true`, generates separate `pyi_deps`. Imports in blocks with the format `if typing.TYPE_CHECKING:`/`if TYPE_CHECKING:` and type-only stub packages (eg. boto3-stubs) are recognized as type-checking dependencies. | -| [`# gazelle:python_generate_proto`](#directive-python_generate_proto) | `false` | -| Controls whether to generate a `py_proto_library` for each `proto_library` in the package. By default we load this rule from the `@protobuf` repository; use `gazelle:map_kind` if you need to load this from somewhere else. | -| `# gazelle:python_resolve_sibling_imports` | `false` | -| Allows absolute imports to be resolved to sibling modules (Python 2's behavior without `absolute_import`). | - -#### Directive: `python_root`: - -Set this directive within the Bazel package that you want to use as the Python root. -For example, if using a `src` dir (as recommended by the [Python Packaging User -Guide][python-packaging-user-guide]), then set this directive in `src/BUILD.bazel`: - -```starlark -# ./src/BUILD.bazel -# Tell gazelle that are python root is the same dir as this Bazel package. -# gazelle:python_root -``` - -Note that the directive does not have any arguments. - -Gazelle will then add the necessary `imports` attribute to all targets that it -generates: - -```starlark -# in ./src/foo/BUILD.bazel -py_libary( - ... - imports = [".."], # Gazelle adds this - ... -) - -# in ./src/foo/bar/BUILD.bazel -py_libary( - ... - imports = ["../.."], # Gazelle adds this - ... -) -``` - -[python-packaging-user-guide]: https://github.com/pypa/packaging.python.org/blob/4c86169a/source/tutorials/packaging-projects.rst - -#### Directive: `python_proto_naming_convention`: - -Set this directive to a string pattern to control how the generated `py_proto_library` targets are named. When generating new `py_proto_library` rules, Gazelle will replace `$proto_name$` in the pattern with the name of the `proto_library` rule, stripping out a trailing `_proto`. For example: - -```starlark -# gazelle:python_generate_proto true -# gazelle:python_proto_naming_convention my_custom_$proto_name$_pattern - -proto_library( - name = "foo_proto", - srcs = ["foo.proto"], -) -``` - -produces the following `py_proto_library` rule: -```starlark -py_proto_library( - name = "my_custom_foo_pattern", - deps = [":foo_proto"], -) -``` - -The default naming convention is `$proto_name$_pb2_py`, so by default in the above example Gazelle would generate `foo_pb2_py`. Any pre-existing rules are left in place and not renamed. - -Note that the Python library will always be imported as `foo_pb2` in Python code, regardless of the naming convention. Also note that Gazelle is currently not able to map said imports, e.g. `import foo_pb2`, to fill in `py_proto_library` targets as dependencies of other rules. See [this issue](https://github.com/bazel-contrib/rules_python/issues/1703). - -#### Directive: `python_default_visibility`: - -Instructs gazelle to use these visibility labels on all _python_ targets -(typically `py_*`, but can be modified via the `map_kind` directive). The arg -to this directive is a a comma-separated list (without spaces) of labels. - -For example: - -```starlark -# gazelle:python_default_visibility //:__subpackages__,//tests:__subpackages__ -``` - -produces the following visibility attribute: - -```starlark -py_library( - ..., - visibility = [ - "//:__subpackages__", - "//tests:__subpackages__", - ], - ..., -) -``` - -You can also inject the `python_root` value by using the exact string -`$python_root$`. All instances of this string will be replaced by the `python_root` -value. - -```starlark -# gazelle:python_default_visibility //$python_root$:__pkg__,//foo/$python_root$/tests:__subpackages__ - -# Assuming the "# gazelle:python_root" directive is set in ./py/src/BUILD.bazel, -# the results will be: -py_library( - ..., - visibility = [ - "//foo/py/src/tests:__subpackages__", # sorted alphabetically - "//py/src:__pkg__", - ], - ..., -) -``` - -Two special values are also accepted as an argument to the directive: - -+ `NONE`: This removes all default visibility. Labels added by the - `python_visibility` directive are still included. -+ `DEFAULT`: This resets the default visibility. - -For example: - -```starlark -# gazelle:python_default_visibility NONE - -py_library( - name = "...", - srcs = [...], -) -``` - -```starlark -# gazelle:python_default_visibility //foo:bar -# gazelle:python_default_visibility DEFAULT - -py_library( - ..., - visibility = ["//:__subpackages__"], - ..., -) -``` - -These special values can be useful for sub-packages. - - -#### Directive: `python_visibility`: - -Appends additional `visibility` labels to each generated target. - -This directive can be set multiple times. The generated `visibility` attribute -will include the default visibility and all labels defined by this directive. -All labels will be ordered alphabetically. - -```starlark -# ./BUILD.bazel -# gazelle:python_visibility //tests:__pkg__ -# gazelle:python_visibility //bar:baz - -py_library( - ... - visibility = [ - "//:__subpackages__", # default visibility - "//bar:baz", - "//tests:__pkg__", - ], - ... -) -``` - -Child Bazel packages inherit values from parents: - -```starlark -# ./bar/BUILD.bazel -# gazelle:python_visibility //tests:__subpackages__ - -py_library( - ... - visibility = [ - "//:__subpackages__", # default visibility - "//bar:baz", # defined in ../BUILD.bazel - "//tests:__pkg__", # defined in ../BUILD.bazel - "//tests:__subpackages__", # defined in this ./BUILD.bazel - ], - ... -) - -``` - -This directive also supports the `$python_root$` placeholder that -`# gazelle:python_default_visibility` supports. - -```starlark -# gazlle:python_visibility //$python_root$/foo:bar - -py_library( - ... - visibility = ["//this_is_my_python_root/foo:bar"], - ... -) -``` - - -#### Directive: `python_test_file_pattern`: - -This directive adjusts which python files will be mapped to the `py_test` rule. - -+ The default is `*_test.py,test_*.py`: both `test_*.py` and `*_test.py` files - will generate `py_test` targets. -+ This directive must have a value. If no value is given, an error will be raised. -+ It is recommended, though not necessary, to include the `.py` extension in - the `glob`s: `foo*.py,?at.py`. -+ Like most directives, it applies to the current Bazel package and all subpackages - until the directive is set again. -+ This directive accepts multiple `glob` patterns, separated by commas without spaces: - -```starlark -# gazelle:python_test_file_pattern foo*.py,?at - -py_library( - name = "mylib", - srcs = ["mylib.py"], -) - -py_test( - name = "foo_bar", - srcs = ["foo_bar.py"], -) - -py_test( - name = "cat", - srcs = ["cat.py"], -) - -py_test( - name = "hat", - srcs = ["hat.py"], -) -``` - - -##### Notes - -Resetting to the default value (such as in a subpackage) is manual. Set: - -```starlark -# gazelle:python_test_file_pattern *_test.py,test_*.py -``` - -There currently is no way to tell gazelle that _no_ files in a package should -be mapped to `py_test` targets (see [Issue #1826][issue-1826]). The workaround -is to set this directive to a pattern that will never match a `.py` file, such -as `foo.bar`: - -```starlark -# No files in this package should be mapped to py_test targets. -# gazelle:python_test_file_pattern foo.bar - -py_library( - name = "my_test", - srcs = ["my_test.py"], -) -``` - -[issue-1826]: https://github.com/bazel-contrib/rules_python/issues/1826 - -#### Directive: `python_generation_mode_per_package_require_test_entry_point`: -When `# gazelle:python_generation_mode package`, whether a file called `__test__.py` or a target called `__test__`, a.k.a., entry point, is required to generate one test target per package. If this is set to true but no entry point is found, Gazelle will fall back to file mode and generate one test target per file. Setting this directive to false forces Gazelle to generate one test target per package even without entry point. However, this means the `main` attribute of the `py_test` will not be set and the target will not be runnable unless either: -1. there happen to be a file in the `srcs` with the same name as the `py_test` target, or -2. a macro populating the `main` attribute of `py_test` is configured with `gazelle:map_kind` to replace `py_test` when Gazelle is generating Python test targets. For example, user can provide such a macro to Gazelle: - -```starlark -load("@rules_python//python:defs.bzl", _py_test="py_test") -load("@aspect_rules_py//py:defs.bzl", "py_pytest_main") - -def py_test(name, main=None, **kwargs): - deps = kwargs.pop("deps", []) - if not main: - py_pytest_main( - name = "__test__", - deps = ["@pip_pytest//:pkg"], # change this to the pytest target in your repo. - ) - - deps.append(":__test__") - main = ":__test__.py" - - _py_test( - name = name, - main = main, - deps = deps, - **kwargs, -) -``` - -#### Directive: `python_generate_proto`: - -When `# gazelle:python_generate_proto true`, Gazelle will generate one -`py_proto_library` for each `proto_library`, generating Python clients for -protobuf in each package. By default this is turned off. Gazelle will also -generate a load statement for the `py_proto_library` - attempting to detect -the configured name for the `@protobuf` / `@com_google_protobuf` repo in your -`MODULE.bazel`, and otherwise falling back to `@com_google_protobuf` for -compatibility with `WORKSPACE`. - -For example, in a package with `# gazelle:python_generate_proto true` and a -`foo.proto`, if you have both the proto extension and the Python extension -loaded into Gazelle, you'll get something like: - -```starlark -load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library") -load("@rules_proto//proto:defs.bzl", "proto_library") - -# gazelle:python_generate_proto true - -proto_library( - name = "foo_proto", - srcs = ["foo.proto"], - visibility = ["//:__subpackages__"], -) - -py_proto_library( - name = "foo_py_pb2", - visibility = ["//:__subpackages__"], - deps = [":foo_proto"], -) -``` - -When `false`, Gazelle will ignore any `py_proto_library`, including previously-generated or hand-created rules. - - -#### Directive: `python_experimental_allow_relative_imports` -Enables experimental support for resolving relative imports in -`python_generation_mode package`. - -By default, when `# gazelle:python_generation_mode package` is enabled, -relative imports (e.g., from .library import foo) are not added to the -deps field of the generated target. This results in incomplete py_library -rules that lack required dependencies on sibling packages. - -Example: -Given this Python file import: -```python -from .library import add as _add -from .library import subtract as _subtract -``` - -Expected BUILD file output: -```starlark -py_library( - name = "py_default_library", - srcs = ["__init__.py"], - deps = [ - "//example/library:py_default_library", - ], - visibility = ["//visibility:public"], -) -``` - -Actual output without this annotation: -```starlark -py_library( - name = "py_default_library", - srcs = ["__init__.py"], - visibility = ["//visibility:public"], -) -``` -If the directive is set to `true`, gazelle will resolve imports -that are relative to the current package. - ### Libraries Python source files are those ending in `.py` but not ending in `_test.py`. diff --git a/gazelle/docs/directives.md b/gazelle/docs/directives.md new file mode 100644 index 0000000000..9221c60823 --- /dev/null +++ b/gazelle/docs/directives.md @@ -0,0 +1,647 @@ +# Directives + +You can configure the extension using directives, just like for other +languages. These are just comments in the `BUILD.bazel` file which +govern behavior of the extension when processing files under that +folder. + +See the [Gazelle docs on directives][gazelle-directives] for some general +directives that may be useful. In particular, the `resolve` directive +is language-specific and can be used with Python. Examples of these and +the Python-specific directives in use can be found in the +{gh-path}`gazelle/testdata` folder in the `rules_python` repo. + +[gazelle-directives]: https://github.com/bazelbuild/bazel-gazelle#directives + +The Python-specific directives are: + +* [`# gazelle:python_extension`](#python-extension) + * Default: `enabled` + * Allowed Values: `enabled`, `disabled` + * Controls whether the Python extension is enabled or not. Sub-packages + inherit this value. +* [`# gazelle:python_root`](#python-root) + * Default: n/a + * Allowed Values: None. This direcive does not consume values. + * Sets a Bazel package as a Python root. This is used on monorepos with + multiple Python projects that don't share the top-level of the workspace + as the root. +* [`# gazelle:python_manifest_file_name`](#python-manifest-file-name) + * Default: `gazelle_python.yaml` + * Allowed Values: A string + * Overrides the default manifest file name. +* [`# gazelle:python_ignore_files`](#python-ignore-files) + * Default: n/a + * Allowed Values: WIP + * Controls the files which are ignored from the generated targets. +* [`# gazelle:python_ignore_dependencies`](#python-ignore-dependencies) + * Default: n/a + * Allowed Values: WIP + * Controls the ignored dependencies from the generated targets. +* [`# gazelle:python_validate_import_statements`](#python-validate-import-statements) + * Default: `true` + * Allowed Values: `true`, `false` + * Controls whether the Python import statements should be validated. +* [`# gazelle:python_generation_mode`](#python-generation-mode) + * Default: `package` + * Allowed Values: `file`, `package`, `project` + * Controls the target generation mode. +* [`# gazelle:python_generation_mode_per_file_include_init`](#python-generation-mode-per-file-include-init) + * Default: `false` + * Allowed Values: `true`, `false` + * Controls whether `__init__.py` files are included as srcs in each + generated target when target generation mode is "file". +* [`# gazelle:python_generation_mode_per_package_require_test_entry_point`](python-generation-mode-per-package-require-test-entry-point) + * Default: `true` + * Allowed Values: `true`, `false` + * Controls whether a file called `__test__.py` or a target called + `__test__` is required to generate one test target per package in + package mode. +* [`# gazelle:python_library_naming_convention`](#python-library-naming-convention) + * Default: `$package_name$` + * Allowed Values: A string containing `"$package_name$"` + * Controls the {bzl:obj}`py_library` naming convention. It interpolates + `$package_name$` with the Bazel package name. E.g. if the Bazel package + name is `foo`, setting this to `$package_name$_my_lib` would result in a + generated target named `foo_my_lib`. +* [`# gazelle:python_binary_naming_convention`](#python-binary-naming-convention) + * Default: `$package_name$_bin` + * Allowed Values: A string containing `"$package_name$"` + * Controls the {bzl:obj}`py_binary` naming convention. Follows the same interpolation + rules as `python_library_naming_convention`. +* [`# gazelle:python_test_naming_convention`](#python-test-naming-convention) + * Default: `$package_name$_test` + * Allowed Values: A string containing `"$package_name$"` + * Controls the {bzl:obj}`py_test` naming convention. Follows the same interpolation + rules as `python_library_naming_convention`. +* [`# gazelle:python_proto_naming_convention`](#python-proto-naming-convention) + * Default: `$proto_name$_py_pb2` + * Allowed Values: A string containing `"$proto_name$"` + * Controls the {bzl:obj}`py_proto_library` naming convention. It interpolates + `$proto_name$` with the {bzl:obj}`proto_library` rule name, minus any trailing + `_proto`. E.g. if the {bzl:obj}`proto_library` name is `foo_proto`, setting this + to `$proto_name$_my_lib` would render to `foo_my_lib`. +* [`# gazelle:resolve py ...`](#resolve-py) + * Default: n/a + * Allowed Values: See the [bazel-gazelle docs][gazelle-directives] + * Instructs the plugin what target to add as a dependency to satisfy a given + import statement. The syntax is `# gazelle:resolve py import-string label` + where `import-string` is the symbol in the python `import` statement, + and `label` is the Bazel label that Gazelle should write in `deps`. +* [`# gazelle:python_default_visibility labels`](python-default-visibility) + * Default: `//$python_root$:__subpackages__` + * Allowed Values: A string + * Instructs gazelle to use these visibility labels on all python targets. + `labels` is a comma-separated list of labels (without spaces). +* [`# gazelle:python_visibility label`](python-visibility) + * Default: n/a + * Allowed Values: A string + * Appends additional visibility labels to each generated target. This r + directive can be set multiple times. +* [`# gazelle:python_test_file_pattern`](python-test-file-pattern) + * Default: `*_test.py,test_*.py` + * Allowed Values: A glob string + * Filenames matching these comma-separated {command}`glob`s will be mapped to + {bzl:obj}`py_test` targets. +* [`# gazelle:python_label_convention`](#python-label-convention) + * Default: `$distribution_name$` + * Allowed Values: A string + * Defines the format of the distribution name in labels to third-party deps. + Useful for using Gazelle plugin with other rules with different repository + conventions (e.g. `rules_pycross`). Full label is always prepended with + the `pip` repository name, e.g. `@pip//numpy` if your + `MODULE.bazel` has `use_repo(pip, "pip")` or `@pypi//numpy` + if your `MODULE.bazel` has `use_repo(pip, "pypi")`. +* [`# gazelle:python_label_normalization`](#python-label-normalization) + * Default: `snake_case` + * Allowed Values: `snake_case`, `none`, `pep503` + * Controls how distribution names in labels to third-party deps are + normalized. Useful for using Gazelle plugin with other rules with different + label conventions (e.g. `rules_pycross` uses PEP-503). +* [`# gazelle:python_experimental_allow_relative_imports`](#python-experimental-allow-relative-imports) + * Default: `false` + * Allowed Values: `true`, `false` + * Controls whether Gazelle resolves dependencies for import statements that + use paths relative to the current package. +* [`# gazelle:python_generate_pyi_deps`](#python-generate-pyi-deps) + * Default: `false` + * Allowed Values: `true`, `false` + * Controls whether to generate a separate `pyi_deps` attribute for + type-checking dependencies or merge them into the regular `deps` + attribute. When `false` (default), type-checking dependencies are + merged into `deps` for backward compatibility. When `true`, generates + separate `pyi_deps`. Imports in blocks with the format + `if typing.TYPE_CHECKING:` or `if TYPE_CHECKING:` and type-only stub + packages (eg. boto3-stubs) are recognized as type-checking dependencies. +* [`# gazelle:python_generate_proto`](#python-generate-proto) + * Default: `false` + * Allowed Values: `true`, `false` + * Controls whether to generate a {bzl:obj}`py_proto_library` for each + {bzl:obj}`proto_library` in the package. By default we load this rule from the + `@protobuf` repository; use `gazelle:map_kind` if you need to load this + from somewhere else. +* [`# gazelle:python_resolve_sibling_imports`](#python-resolve-sibling-imports) + * Default: `false` + * Allowed Values: `true`, `false` + * Allows absolute imports to be resolved to sibling modules (Python 2's + behavior without `absolute_import`). + + +## `python_extension` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_root` + +Set this directive within the Bazel package that you want to use as the Python root. +For example, if using a `src` dir (as recommended by the [Python Packaging User +Guide][python-packaging-user-guide]), then set this directive in `src/BUILD.bazel`: + +```starlark +# ./src/BUILD.bazel +# Tell gazelle that are python root is the same dir as this Bazel package. +# gazelle:python_root +``` + +Note that the directive does not have any arguments. + +Gazelle will then add the necessary `imports` attribute to all targets that it +generates: + +```starlark +# in ./src/foo/BUILD.bazel +py_libary( + ... + imports = [".."], # Gazelle adds this + ... +) + +# in ./src/foo/bar/BUILD.bazel +py_libary( + ... + imports = ["../.."], # Gazelle adds this + ... +) +``` + +[python-packaging-user-guide]: https://github.com/pypa/packaging.python.org/blob/4c86169a/source/tutorials/packaging-projects.rst + + +## `python_manifest_file_name` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_ignore_files` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_ignore_dependencies` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_validate_import_statements` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_generation_mode` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_generation_mode_per_file_include_init` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_generation_mode_per_package_require_test_entry_point` + +When `# gazelle:python_generation_mode package`, whether a file called +`__test__.py` or a target called `__test__`, a.k.a., entry point, is required +to generate one test target per package. If this is set to true but no entry +point is found, Gazelle will fall back to file mode and generate one test target +per file. Setting this directive to false forces Gazelle to generate one test +target per package even without entry point. However, this means the `main` +attribute of the {bzl:obj}`py_test` will not be set and the target will not be runnable +unless either: + +1. there happen to be a file in the `srcs` with the same name as the {bzl:obj}`py_test` + target, or +2. a macro populating the `main` attribute of {bzl:obj}`py_test` is configured with + `gazelle:map_kind` to replace {bzl:obj}`py_test` when Gazelle is generating Python + test targets. For example, user can provide such a macro to Gazelle: + +```starlark +load("@rules_python//python:defs.bzl", _py_test="py_test") +load("@aspect_rules_py//py:defs.bzl", "py_pytest_main") + +def py_test(name, main=None, **kwargs): + deps = kwargs.pop("deps", []) + if not main: + py_pytest_main( + name = "__test__", + deps = ["@pip_pytest//:pkg"], # change this to the pytest target in your repo. + ) + + deps.append(":__test__") + main = ":__test__.py" + + _py_test( + name = name, + main = main, + deps = deps, + **kwargs, +) +``` + + +## `python_library_naming_convention` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_binary_naming_convention` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_test_naming_convention` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_proto_naming_convention` + +Set this directive to a string pattern to control how the generated +{bzl:obj}`py_proto_library` targets are named. When generating new +{bzl:obj}`py_proto_library` rules, Gazelle will replace `$proto_name$` in the +pattern with the name of the {bzl:obj}`proto_library` rule, stripping out a +trailing `_proto`. For example: + +```starlark +# gazelle:python_generate_proto true +# gazelle:python_proto_naming_convention my_custom_$proto_name$_pattern + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) +``` + +produces the following {bzl:obj}`py_proto_library` rule: + +```starlark +py_proto_library( + name = "my_custom_foo_pattern", + deps = [":foo_proto"], +) +``` + +The default naming convention is `$proto_name$_pb2_py` in accordance with +the [Bazel `py_proto_library` convention][bazel-py-proto-library], so by default +in the above example Gazelle would generate `foo_pb2_py`. Any pre-existing +rules are left in place and not renamed. + +[bazel-py-proto-library]: https://bazel.build/reference/be/protocol-buffer#py_proto_library + +Note that the Python library will always be imported as `foo_pb2` in Python +code, regardless of the naming convention. Also note that Gazelle is currently +not able to map said imports, e.g. `import foo_pb2`, to fill in +{bzl:obj}`py_proto_library` targets as dependencies of other rules. See +{gh-issue}`1703`. + + +## `resolve py` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_default_visibility` + +Instructs gazelle to use these visibility labels on all _python_ targets +(typically `py_*`, but can be modified via the `map_kind` directive). The arg +to this directive is a comma-separated list (without spaces) of labels. + +For example: + +```starlark +# gazelle:python_default_visibility //:__subpackages__,//tests:__subpackages__ +``` + +produces the following visibility attribute: + +```starlark +py_library( + ..., + visibility = [ + "//:__subpackages__", + "//tests:__subpackages__", + ], + ..., +) +``` + +You can also inject the `python_root` value by using the exact string +`$python_root$`. All instances of this string will be replaced by the `python_root` +value. + +```starlark +# gazelle:python_default_visibility //$python_root$:__pkg__,//foo/$python_root$/tests:__subpackages__ + +# Assuming the "# gazelle:python_root" directive is set in ./py/src/BUILD.bazel, +# the results will be: +py_library( + ..., + visibility = [ + "//foo/py/src/tests:__subpackages__", # sorted alphabetically + "//py/src:__pkg__", + ], + ..., +) +``` + +Two special values are also accepted as an argument to the directive: + +* `NONE`: This removes all default visibility. Labels added by the + `python_visibility` directive are still included. +* `DEFAULT`: This resets the default visibility. + +For example: + +```starlark +# gazelle:python_default_visibility NONE + +py_library( + name = "...", + srcs = [...], +) +``` + +```starlark +# gazelle:python_default_visibility //foo:bar +# gazelle:python_default_visibility DEFAULT + +py_library( + ..., + visibility = ["//:__subpackages__"], + ..., +) +``` + +These special values can be useful for sub-packages. + + +## `python_visibility` + +Appends additional `visibility` labels to each generated target. + +This directive can be set multiple times. The generated `visibility` attribute +will include the default visibility and all labels defined by this directive. +All labels will be ordered alphabetically. + +```starlark +# ./BUILD.bazel +# gazelle:python_visibility //tests:__pkg__ +# gazelle:python_visibility //bar:baz + +py_library( + ... + visibility = [ + "//:__subpackages__", # default visibility + "//bar:baz", + "//tests:__pkg__", + ], + ... +) +``` + +Child Bazel packages inherit values from parents: + +```starlark +# ./bar/BUILD.bazel +# gazelle:python_visibility //tests:__subpackages__ + +py_library( + ... + visibility = [ + "//:__subpackages__", # default visibility + "//bar:baz", # defined in ../BUILD.bazel + "//tests:__pkg__", # defined in ../BUILD.bazel + "//tests:__subpackages__", # defined in this ./BUILD.bazel + ], + ... +) + +``` + +This directive also supports the `$python_root$` placeholder that +`# gazelle:python_default_visibility` supports. + +```starlark +# gazlle:python_visibility //$python_root$/foo:bar + +py_library( + ... + visibility = ["//this_is_my_python_root/foo:bar"], + ... +) +``` + + +## `python_test_file_pattern` + +This directive adjusts which python files will be mapped to the {bzl:obj}`py_test` rule. + ++ The default is `*_test.py,test_*.py`: both `test_*.py` and `*_test.py` files + will generate {bzl:obj}`py_test` targets. ++ This directive must have a value. If no value is given, an error will be raised. ++ It is recommended, though not necessary, to include the `.py` extension in + the {command}`glob`: `foo*.py,?at.py`. ++ Like most directives, it applies to the current Bazel package and all subpackages + until the directive is set again. ++ This directive accepts multiple {command}`glob` patterns, separated by commas without spaces: + +```starlark +# gazelle:python_test_file_pattern foo*.py,?at + +py_library( + name = "mylib", + srcs = ["mylib.py"], +) + +py_test( + name = "foo_bar", + srcs = ["foo_bar.py"], +) + +py_test( + name = "cat", + srcs = ["cat.py"], +) + +py_test( + name = "hat", + srcs = ["hat.py"], +) +``` + + +### Notes + +Resetting to the default value (such as in a subpackage) is manual. Set: + +```starlark +# gazelle:python_test_file_pattern *_test.py,test_*.py +``` + +There currently is no way to tell gazelle that _no_ files in a package should +be mapped to {bzl:obj}`py_test` targets (see {gh-issue}`1826`). The workaround +is to set this directive to a pattern that will never match a `.py` file, such +as `foo.bar`: + +```starlark +# No files in this package should be mapped to py_test targets. +# gazelle:python_test_file_pattern foo.bar + +py_library( + name = "my_test", + srcs = ["my_test.py"], +) +``` + + +## `python_label_convention` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_label_normalization` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_experimental_allow_relative_imports` + +Enables experimental support for resolving relative imports in +`python_generation_mode package`. + +By default, when `# gazelle:python_generation_mode package` is enabled, +relative imports (e.g., `from .library import foo`) are not added to the +deps field of the generated target. This results in incomplete {bzl:obj}`py_library` +rules that lack required dependencies on sibling packages. + +Example: + +Given this Python file import: + +```python +from .library import add as _add +from .library import subtract as _subtract +``` + +Expected BUILD file output: + +```starlark +py_library( + name = "py_default_library", + srcs = ["__init__.py"], + deps = [ + "//example/library:py_default_library", + ], + visibility = ["//visibility:public"], +) +``` + +Actual output without this annotation: + +```starlark +py_library( + name = "py_default_library", + srcs = ["__init__.py"], + visibility = ["//visibility:public"], +) +``` + +If the directive is set to `true`, gazelle will resolve imports +that are relative to the current package. + + +## `python_generate_pyi_deps` + +:::{error} +Detailed docs are not yet written. +::: + + +## `python_generate_proto` + +When `# gazelle:python_generate_proto true`, Gazelle will generate one +{bzl:obj}`py_proto_library` for each {bzl:obj}`proto_library`, generating Python clients for +protobuf in each package. By default this is turned off. Gazelle will also +generate a load statement for the {bzl:obj}`py_proto_library` - attempting to detect +the configured name for the `@protobuf` / `@com_google_protobuf` repo in your +`MODULE.bazel`, and otherwise falling back to `@com_google_protobuf` for +compatibility with `WORKSPACE`. + +For example, in a package with `# gazelle:python_generate_proto true` and a +`foo.proto`, if you have both the proto extension and the Python extension +loaded into Gazelle, you'll get something like: + +```starlark +load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") + +# gazelle:python_generate_proto true + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], + visibility = ["//:__subpackages__"], +) + +py_proto_library( + name = "foo_py_pb2", + visibility = ["//:__subpackages__"], + deps = [":foo_proto"], +) +``` + +When `false`, Gazelle will ignore any {bzl:obj}`py_proto_library`, including +previously-generated or hand-created rules. + + +## `python_resolve_sibling_imports` + +:::{error} +Detailed docs are not yet written. +::: diff --git a/gazelle/docs/index.md b/gazelle/docs/index.md index e262d7ff46..6758e11d81 100644 --- a/gazelle/docs/index.md +++ b/gazelle/docs/index.md @@ -43,5 +43,6 @@ the `update` command (the default) does anything for Python code. ```{toctree} :maxdepth: 1 installation_and_usage +directives annotations ``` From ae2fee13c192440be166fa2107e44b3555510100 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 6 Aug 2025 09:12:42 -0700 Subject: [PATCH 26/28] docs(gazelle): Migrate Gazelle docs to ReadTheDocs, part 5/5: target types (#3147) Part of #3082 5th of ~~5~~ 6 PRs. + Migrate target types docs from `gazelle/README.md` to `gazelle/docs/installation_and_usage.md` + `Libraries` section: + Update wording + list for the generation mode types + `Binaries` section: + Slight rewording + `note` block instead of simple "Note that ..." + Mechanical updates: + Wrap at ~80 chars + Use MyST directives and roles. --- gazelle/README.md | 70 ---------------------- gazelle/docs/installation_and_usage.md | 83 ++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 70 deletions(-) diff --git a/gazelle/README.md b/gazelle/README.md index 4de2c3c0cd..30067b4c23 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -6,76 +6,6 @@ ReadTheDocs. Please see https://rules-python.readthedocs.io/gazelle/docs/index.h ::: -### Libraries - -Python source files are those ending in `.py` but not ending in `_test.py`. - -First, we look for the nearest ancestor BUILD file starting from the folder -containing the Python source file. - -In package generation mode, if there is no `py_library` in this BUILD file, one -is created using the package name as the target's name. This makes it the -default target in the package. Next, all source files are collected into the -`srcs` of the `py_library`. - -In project generation mode, all source files in subdirectories (that don't have -BUILD files) are also collected. - -In file generation mode, each file is given its own target. - -Finally, the `import` statements in the source files are parsed, and -dependencies are added to the `deps` attribute. - -### Unit Tests - -A `py_test` target is added to the BUILD file when gazelle encounters -a file named `__test__.py`. -Often, Python unit test files are named with the suffix `_test`. -For example, if we had a folder that is a package named "foo" we could have a Python file named `foo_test.py` -and gazelle would create a `py_test` block for the file. - -The following is an example of a `py_test` target that gazelle would add when -it encounters a file named `__test__.py`. - -```starlark -py_test( - name = "build_file_generation_test", - srcs = ["__test__.py"], - main = "__test__.py", - deps = [":build_file_generation"], -) -``` - -You can control the naming convention for test targets by adding a gazelle directive named -`# gazelle:python_test_naming_convention`. See the instructions in the section above that -covers directives. - -### Binaries - -When a `__main__.py` file is encountered, this indicates the entry point -of a Python program. A `py_binary` target will be created, named `[package]_bin`. - -When no such entry point exists, Gazelle will look for a line like this in the top level in every module: - -```python -if __name == "__main__": -``` - -Gazelle will create a `py_binary` target for every module with such a line, with -the target name the same as the module name. - -If `python_generation_mode` is set to `file`, then instead of one `py_binary` -target per module, Gazelle will create one `py_binary` target for each file with -such a line, and the name of the target will match the name of the script. - -Note that it's possible for another script to depend on a `py_binary` target and -import from the `py_binary`'s scripts. This can have possible negative effects on -Bazel analysis time and runfiles size compared to depending on a `py_library` -target. The simplest way to avoid these negative effects is to extract library -code into a separate script without a `main` line. Gazelle will then create a -`py_library` target for that library code, and other scripts can depend on that -`py_library` target. - ## Developer Notes Gazelle extensions are written in Go. diff --git a/gazelle/docs/installation_and_usage.md b/gazelle/docs/installation_and_usage.md index e764957581..123f30a068 100644 --- a/gazelle/docs/installation_and_usage.md +++ b/gazelle/docs/installation_and_usage.md @@ -149,3 +149,86 @@ gazelle( That's it, now you can finally run `bazel run //:gazelle` anytime you edit Python code, and it should update your `BUILD` files correctly. + + +## Target Types and How They're Generated + +### Libraries + +Python source files are those ending in `.py` that are not matched as a test +file via the `# gazelle:python_test_file_pattern` directive. By default, +python source files are all `*.py` files except for `*_test.py` and +`test_*.py`. + +First, we look for the nearest ancestor `BUILD(.bazel)` file starting from +the folder containing the Python source file. + ++ In `package` generation mode, if there is no {bzl:obj}`py_library` in this + `BUILD(.bazel)` file, one is created using the package name as the target's + name. This makes it the default target in the package. Next, all source + files are collected into the `srcs` of the {bzl:obj}`py_library`. ++ In `project` generation mode, all source files in subdirectories (that don't + have `BUILD(.bazel)` files) are also collected. ++ In `file` generation mode, each python source file is given its own target. + +Finally, the `import` statements in the source files are parsed and +dependencies are added to the `deps` attribute of the target. + + +### Tests + +A {bzl:obj}`py_test` target is added to the `BUILD(.bazel)` file when gazelle +encounters a file named `__test__.py` or when files matching the +`# gazelle:python_test_file_pattern` directive are found. + +For example, if we had a folder that is a package named "foo" we could have a +Python file named `foo_test.py` and gazelle would create a {bzl:obj}`py_test` +block for the file. + +The following is an example of a {bzl:obj}`py_test` target that gazelle would +add when it encounters a file named `__test__.py`. + +```starlark +py_test( + name = "build_file_generation_test", + srcs = ["__test__.py"], + main = "__test__.py", + deps = [":build_file_generation"], +) +``` + +You can control the naming convention for test targets using the +`# gazelle:python_test_naming_convention` directive. + + +### Binaries + +When a `__main__.py` file is encountered, this indicates the entry point +of a Python program. A {bzl:obj}`py_binary` target will be created, named +`[package]_bin`. + +When no such entry point exists, Gazelle will look for a line like this in +the top level in every module: + +```python +if __name == "__main__": +``` + +Gazelle will create a {bzl:obj}`py_binary` target for every module with such +a line, with the target name the same as the module name. + +If the `# gazelle:python_generation_mode` directive is set to `file`, then +instead of one {bzl:obj}`py_binary` target per module, Gazelle will create +one {bzl:obj}`py_binary` target for each file with such a line, and the name +of the target will match the name of the script. + +:::{note} +It's possible for another script to depend on a {bzl:obj}`py_binary` target +and import from the {bzl:obj}`py_binary`'s scripts. This can have possible +negative effects on Bazel analysis time and runfiles size compared to +depending on a {bzl:obj}`py_library` target. The simplest way to avoid these +negative effects is to extract library code into a separate script without a +`main` line. Gazelle will then create a {bzl:obj}`py_library` target for +that library code, and other scripts can depend on that {bzl:obj}`py_library` +target. +::: From e3655c2bee36225366a89554a5a2628237c84f00 Mon Sep 17 00:00:00 2001 From: Douglas Thor Date: Wed, 6 Aug 2025 09:31:06 -0700 Subject: [PATCH 27/28] docs(gazelle): Migrate Gazelle docs to ReadTheDocs, part 6/5: development (#3149) Fixes #3082 6th of 5 PRs. + Migrate gazelle development docs from `gazelle/README.md` to `gazelle/docs/development.md` + Add information on writing tests + Mechanical updates: + Wrap at ~80 chars + Use MyST directives and roles. --- gazelle/README.md | 18 +----------- gazelle/docs/development.md | 57 +++++++++++++++++++++++++++++++++++++ gazelle/docs/index.md | 1 + 3 files changed, 59 insertions(+), 17 deletions(-) create mode 100644 gazelle/docs/development.md diff --git a/gazelle/README.md b/gazelle/README.md index 30067b4c23..128fb1f583 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -1,22 +1,6 @@ # Python Gazelle plugin :::{note} -The gazelle plugin docs are being migrated to our primary documentation on +The gazelle plugin docs have been migrated to our primary documentation on ReadTheDocs. Please see https://rules-python.readthedocs.io/gazelle/docs/index.html. ::: - - -## Developer Notes - -Gazelle extensions are written in Go. -See the gazelle documentation https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.md -for more information on extending Gazelle. - -If you add new Go dependencies to the plugin source code, you need to "tidy" the go.mod file. -After changing that file, run `go mod tidy` or `bazel run @go_sdk//:bin/go -- mod tidy` -to update the go.mod and go.sum files. Then run `bazel run //:gazelle_update_repos` to have gazelle -add the new dependenies to the deps.bzl file. The deps.bzl file is used as defined in our /WORKSPACE -to include the external repos Bazel loads Go dependencies from. - -Then after editing Go code, run `bazel run //:gazelle` to generate/update the rules in the -BUILD.bazel files in our repo. diff --git a/gazelle/docs/development.md b/gazelle/docs/development.md new file mode 100644 index 0000000000..29ac7a0605 --- /dev/null +++ b/gazelle/docs/development.md @@ -0,0 +1,57 @@ +# Development + +Gazelle extensions are written in Go. + +See the [Gazelle documentation][gazelle-extend] for more information on +extending Gazelle. + +[gazelle-extend]: https://github.com/bazel-contrib/bazel-gazelle/blob/master/extend.md + + +## Dependencies + +If you add new Go dependencies to the plugin source code, you need to "tidy" +the go.mod file. After changing that file, run `go mod tidy` or +`bazel run @go_sdk//:bin/go -- mod tidy` to update the `go.mod` and `go.sum` +files. Then run `bazel run //:gazelle_update_repos` to have gazelle add the +new dependencies to the `deps.bzl` file. The `deps.bzl` file is used as +defined in our `/WORKSPACE` to include the external repos Bazel loads Go +dependencies from. + +Then after editing Go code, run `bazel run //:gazelle` to generate/update +the rules in the `BUILD.bazel` files in our repo. + + +## Tests + +:::{seealso} +{gh-path}`gazelle/python/testdata/README.md` +::: + +To run tests, {command}`cd` into the {gh-path}`gazelle` directory and run +`bazel test //...`. + +Test cases are found at {gh-path}`gazelle/python/testdata`. To make a new +test case, create a directory in that folder with the following files: + ++ `README.md` with a short blurb describing the test case(s). ++ `test.yaml`, either empty (with just the docstart `---` line) or with + the expected `stderr` and exit codes of the test case. ++ and empty `WORKSPACE` file + +You will also need `BUILD.in` and `BUILD.out` files somewhere within the test +case directory. These can be in the test case root, in subdirectories, or +both. + ++ `BUILD.in` files are populated with the "before" information - typically + things like Gazelle directives or pre-existing targets. This is how the + `BUILD.bazel` file looks before running Gazelle. ++ `BUILD.out` files are the expected result after running Gazelle within + the test case. + +:::{tip} +The easiest way to create a new test is to look at one of the existing test +cases. +::: + +The source code for running tests is {gh-path}`gazelle/python/python_test.go`. diff --git a/gazelle/docs/index.md b/gazelle/docs/index.md index 6758e11d81..f276b0ca16 100644 --- a/gazelle/docs/index.md +++ b/gazelle/docs/index.md @@ -45,4 +45,5 @@ the `update` command (the default) does anything for Python code. installation_and_usage directives annotations +development ``` From 32d7a24d45eae7430b38733353e3ee77583d2da8 Mon Sep 17 00:00:00 2001 From: Malte Poll <1780588+malt3@users.noreply.github.com> Date: Thu, 7 Aug 2025 03:39:18 +0200 Subject: [PATCH 28/28] fix: use "command -v" to find interpreter in $PATH (#3150) In some environments, `which` doesn't work correctly under Bazel, while `command -v` does. I think the difference is that `command` is a shell builtin (and POSIX compliant), whereas `which` is not: ``` $ sh -c 'builtin command -v python3' /usr/bin/python3 $ sh -c 'builtin which python3' sh: line 1: builtin: which: not a shell builtin ``` While `command -v` performs fewer checks under the hood, it is more portable. --- CHANGELOG.md | 2 ++ python/private/runtime_env_toolchain_interpreter.sh | 12 +++++------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 422e399026..08991c6107 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -101,6 +101,8 @@ END_UNRELEASED_TEMPLATE `# gazelle:python_resolve_sibling_imports true` * (pypi) Show overridden index URL of packages when downloading metadata have failed. ([#2985](https://github.com/bazel-contrib/rules_python/issues/2985)). +* (toolchains) use "command -v" to find interpreter in `$PATH` + ([#3150](https://github.com/bazel-contrib/rules_python/pull/3150)). {#v0-0-0-added} ### Added diff --git a/python/private/runtime_env_toolchain_interpreter.sh b/python/private/runtime_env_toolchain_interpreter.sh index dd4d648d12..c78cfe1a9b 100755 --- a/python/private/runtime_env_toolchain_interpreter.sh +++ b/python/private/runtime_env_toolchain_interpreter.sh @@ -17,16 +17,14 @@ die() { exit 1 } -# We use `which` to locate the Python interpreter command on PATH. `command -v` -# is another option, but it doesn't check whether the file it finds has the -# executable bit. +# We use `command -v` to locate the Python interpreter command on PATH. # # A tricky situation happens when this wrapper is invoked as part of running a # tool, e.g. passing a py_binary target to `ctx.actions.run()`. Bazel will unset # the PATH variable. Then the shell will see there's no PATH and initialize its -# own, sometimes without exporting it. This causes `which` to fail and this +# own, sometimes without exporting it. This causes `command -v` to fail and this # script to think there's no Python interpreter installed. To avoid this we -# explicitly pass PATH to each `which` invocation. We can't just export PATH +# explicitly pass PATH to each `command -v` invocation. We can't just export PATH # because that would modify the environment seen by the final user Python # program. # @@ -37,9 +35,9 @@ die() { # https://github.com/bazelbuild/bazel/issues/8415 # Try the "python3" command name first, then fall back on "python". -PYTHON_BIN="$(PATH="$PATH" which python3 2> /dev/null)" +PYTHON_BIN="$(PATH="$PATH" command -v python3 2> /dev/null)" if [ -z "${PYTHON_BIN:-}" ]; then - PYTHON_BIN="$(PATH="$PATH" which python 2>/dev/null)" + PYTHON_BIN="$(PATH="$PATH" command -v python 2>/dev/null)" fi if [ -z "${PYTHON_BIN:-}" ]; then die "Neither 'python3' nor 'python' were found on the target \ pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy