Skip to content

feat(gazelle): create py_binary targets for if __name__ == "__main__" #1566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

feat(gazelle): create py_binary targets for if __name__ == "__main__" #1566

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,9 @@ A brief description of the categories of changes:
`__test__.py` exists in the same package. Previously in these cases there
would only be one test target made.

* (gazelle) If a non-test Python file contains `if __name__ == "__main__":`,
then a `py_binary` target is made for it instead of a `py_library` target.

Breaking changes:

* (pip) `pip_install` repository rule in this release has been disabled and
Expand Down
47 changes: 34 additions & 13 deletions gazelle/python/generate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"log"
"os"
"path/filepath"
"regexp"
"strings"

"github.com/bazelbuild/bazel-gazelle/config"
Expand Down Expand Up @@ -85,14 +86,11 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes

packageName := filepath.Base(args.Dir)

pyBinaryFilenames := treeset.NewWith(godsutils.StringComparator)
pyLibraryFilenames := treeset.NewWith(godsutils.StringComparator)
pyTestFilenames := treeset.NewWith(godsutils.StringComparator)
pyFileNames := treeset.NewWith(godsutils.StringComparator)

// hasPyBinary controls whether a py_binary target should be generated for
// this package or not.
hasPyBinary := false

// hasPyTestEntryPointFile and hasPyTestEntryPointTarget control whether a py_test target should
// be generated for this package or not.
hasPyTestEntryPointFile := false
Expand All @@ -106,14 +104,14 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
ext := filepath.Ext(f)
if ext == ".py" {
pyFileNames.Add(f)
if !hasPyBinary && f == pyBinaryEntrypointFilename {
hasPyBinary = true
} else if !hasPyTestEntryPointFile && f == pyTestEntrypointFilename {
if !hasPyTestEntryPointFile && f == pyTestEntrypointFilename {
hasPyTestEntryPointFile = true
} else if f == conftestFilename {
hasConftestFile = true
} else if strings.HasSuffix(f, "_test.py") || strings.HasPrefix(f, "test_") {
pyTestFilenames.Add(f)
} else if f == pyBinaryEntrypointFilename || hasNameEqualsMain(filepath.Join(args.Config.RepoRoot, args.Rel, f)) {
pyBinaryFilenames.Add(f)
} else {
pyLibraryFilenames.Add(f)
}
Expand Down Expand Up @@ -270,13 +268,19 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
appendPyLibrary(pyLibraryFilenames, cfg.RenderLibraryName(packageName))
}

if hasPyBinary {
deps, err := parser.parseSingle(pyBinaryEntrypointFilename)
pyBinaryFilenames.Each(func(index int, filename interface{}) {
entrypointFilename := filename.(string)
deps, err := parser.parseSingle(entrypointFilename)
if err != nil {
log.Fatalf("ERROR: %v\n", err)
}

pyBinaryTargetName := cfg.RenderBinaryName(packageName)
var pyBinaryTargetName string
if entrypointFilename == pyBinaryEntrypointFilename {
pyBinaryTargetName = cfg.RenderBinaryName(packageName)
} else {
pyBinaryTargetName = strings.TrimSuffix(filepath.Base(filename.(string)), ".py")
}

// Check if a target with the same name we are generating already
// exists, and if it is of a different kind from the one we are
Expand All @@ -296,17 +300,20 @@ func (py *Python) GenerateRules(args language.GenerateArgs) language.GenerateRes
}

pyBinaryTarget := newTargetBuilder(pyBinaryKind, pyBinaryTargetName, pythonProjectRoot, args.Rel, pyFileNames).
setMain(pyBinaryEntrypointFilename).
addVisibility(visibility).
addSrc(pyBinaryEntrypointFilename).
addSrc(entrypointFilename).
addModuleDependencies(deps).
generateImportsAttribute()

if entrypointFilename == pyBinaryEntrypointFilename {
pyBinaryTarget.setMain(pyBinaryEntrypointFilename)
}

pyBinary := pyBinaryTarget.build()

result.Gen = append(result.Gen, pyBinary)
result.Imports = append(result.Imports, pyBinary.PrivateAttr(config.GazelleImportsKey))
}
})

var conftest *rule.Rule
if hasConftestFile {
Expand Down Expand Up @@ -463,6 +470,20 @@ func hasEntrypointFile(dir string) bool {
return false
}

// hasNameEqualsMain determines if the file contains 'if __name__ == "__main__"'.
func hasNameEqualsMain(path string) bool {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I personally think that this new feature needs a feature toggle. There should be a gazelle directive that enables the behaviour.

What is more, sometimes developers add if __name__ in order to test the script locally but do not intend to create a binary target for others to consume. I wonder if in those cases we should be able to say in the python file:

if __name__ == "__main__":  # gazelle: ignore
    main()

Copy link
Contributor Author

@adzenith adzenith Nov 17, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A feature toggle is a great call. Let me look into making that work.

As for the ignore, is there any reason you wouldn't want that to get moved to a py_binary? Then you could bazel run it and test it locally. (If the script is already in a test target, then this change won't move it - it only moves scripts from py_library -> py_binary, not from py_test, because you can already bazel run a test.) I guess I'm just curious when you might want an ignore / might want to keep a script out of a py_binary target.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's keep the ignore part out of scope for now. It probably does not need to be supported in the initial version.

searchString := `if __name__ == ['"]__main__['"]:`
bytesContents, err := os.ReadFile(path)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right now we are reading the python files once to get the imports and here it would be for once more to check if there is an if __name__ == "main" which may not scale well in super large repos because we have now twice the number of files to process.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let me see how the read to get the import works. Maybe I can figure out a way to not read it twice.

if err != nil {
return false
}
match, err := regexp.Match(searchString, bytesContents)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Reading from the end of the file line by line in reverse and matching the whole line (stripped) could be faster? If you want to still use regexp, you could at least compile the regexp upfront and store it as a var at the top of the file.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, let me look into making this faster.

if err == nil {
return match
}
return false
}

// isEntrypointFile returns whether the given path is an entrypoint file. The
// given path can be absolute or relative.
func isEntrypointFile(path string) bool {
Expand Down
Empty file added gazelle/python/testdata/binary_targets/BUILD.in
View file
Open in desktop
Empty file.
35 changes: 35 additions & 0 deletions gazelle/python/testdata/binary_targets/BUILD.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")

py_library(
name = "binary_targets",
srcs = [
"bar.py",
"baz.py",
],
visibility = ["//:__subpackages__"],
)

py_binary(
name = "bar_binary",
srcs = ["bar_binary.py"],
visibility = ["//:__subpackages__"],
deps = [":binary_targets"],
)

py_binary(
name = "single_quote_main",
srcs = ["single_quote_main.py"],
visibility = ["//:__subpackages__"],
deps = [":bar_test"],
)

py_test(
name = "bar_test",
srcs = ["bar_test.py"],
deps = [":bar_binary"],
)

py_test(
name = "name_main_test",
srcs = ["name_main_test.py"],
)
4 changes: 4 additions & 0 deletions gazelle/python/testdata/binary_targets/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# Binary targets

This test case generates `py_binary` targets for files containing
`if __name__ == "__main__"`.
1 change: 1 addition & 0 deletions gazelle/python/testdata/binary_targets/WORKSPACE
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# This is a Bazel workspace for the Gazelle test data.
Empty file added gazelle/python/testdata/binary_targets/bar.py
View file
Open in desktop
Empty file.
4 changes: 4 additions & 0 deletions gazelle/python/testdata/binary_targets/bar_binary.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import bar

if __name__ == "__main__":
pass
1 change: 1 addition & 0 deletions gazelle/python/testdata/binary_targets/bar_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
import bar_binary
Empty file added gazelle/python/testdata/binary_targets/baz.py
View file
Open in desktop
Empty file.
3 changes: 3 additions & 0 deletions gazelle/python/testdata/binary_targets/name_main_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# This should make a py_test target because of the filename
if __name__ == "__main__":
pass
4 changes: 4 additions & 0 deletions gazelle/python/testdata/binary_targets/single_quote_main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
import bar_test

if __name__ == '__main__':
pass
1 change: 1 addition & 0 deletions gazelle/python/testdata/binary_targets/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
---
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy