Skip to content

[2.7] bpo-36560: Fix reference leak hunting in regrtest (GH-12744) #12745

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 9, 2019
Merged

[2.7] bpo-36560: Fix reference leak hunting in regrtest (GH-12744) #12745

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 38 additions & 18 deletions Lib/test/regrtest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,8 +529,6 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,

if slaveargs is not None:
args, kwargs = json.loads(slaveargs)
if kwargs['huntrleaks']:
warm_caches()
if testdir:
kwargs['testdir'] = testdir
try:
Expand All @@ -541,9 +539,6 @@ def main(tests=None, testdir=None, verbose=0, quiet=False,
print json.dumps(result)
sys.exit(0)

if huntrleaks:
warm_caches()

good = []
bad = []
skipped = []
Expand Down Expand Up @@ -1332,7 +1327,7 @@ def runtest_inner(test, verbose, quiet, huntrleaks=False, pgo=False, testdir=Non
indirect_test = getattr(the_module, "test_main", None)
if huntrleaks:
refleak = dash_R(the_module, test, indirect_test,
huntrleaks)
huntrleaks, quiet)
else:
if indirect_test is not None:
indirect_test()
Expand Down Expand Up @@ -1425,7 +1420,7 @@ def cleanup_test_droppings(testname, verbose):
print >> sys.stderr, ("%r left behind %s %r and it couldn't be "
"removed: %s" % (testname, kind, name, msg))

def dash_R(the_module, test, indirect_test, huntrleaks):
def dash_R(the_module, test, indirect_test, huntrleaks, quiet):
"""Run a test multiple times, looking for reference leaks.

Returns:
Expand All @@ -1438,6 +1433,10 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
raise Exception("Tracking reference leaks requires a debug build "
"of Python")

# Avoid false positives due to various caches
# filling slowly with random data:
warm_caches()

# Save current values for dash_R_cleanup() to restore.
fs = warnings.filters[:]
ps = copy_reg.dispatch_table.copy()
Expand All @@ -1457,6 +1456,14 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
for obj in abc.__subclasses__() + [abc]:
abcs[obj] = obj._abc_registry.copy()

# bpo-31217: Integer pool to get a single integer object for the same
# value. The pool is used to prevent false alarm when checking for memory
# block leaks. Fill the pool with values in -1000..1000 which are the most
# common (reference, memory block, file descriptor) differences.
int_pool = {value: value for value in range(-1000, 1000)}
def get_pooled_int(value):
return int_pool.setdefault(value, value)

if indirect_test:
def run_the_test():
indirect_test()
Expand All @@ -1467,27 +1474,39 @@ def run_the_test():
deltas = []
nwarmup, ntracked, fname = huntrleaks
fname = os.path.join(support.SAVEDCWD, fname)

# Pre-allocate to ensure that the loop doesn't allocate anything new
repcount = nwarmup + ntracked
rc_deltas = [0] * ntracked
fd_deltas = [0] * ntracked
rc_deltas = [0] * repcount
fd_deltas = [0] * repcount
rep_range = list(range(repcount))

if not quiet:
print >> sys.stderr, "beginning", repcount, "repetitions"
print >> sys.stderr, ("1234567890"*(repcount//10 + 1))[:repcount]

print >> sys.stderr, "beginning", repcount, "repetitions"
print >> sys.stderr, ("1234567890"*(repcount//10 + 1))[:repcount]
dash_R_cleanup(fs, ps, pic, zdc, abcs)

# initialize variables to make pyflakes quiet
rc_before = fd_before = 0
for i in range(repcount):

for i in rep_range:
run_the_test()
sys.stderr.write('.')

if not quiet:
sys.stderr.write('.')

dash_R_cleanup(fs, ps, pic, zdc, abcs)

rc_after = sys.gettotalrefcount()
fd_after = support.fd_count()
if i >= nwarmup:
rc_deltas[i - nwarmup] = rc_after - rc_before
fd_deltas[i - nwarmup] = fd_after - fd_before
rc_deltas[i] = get_pooled_int(rc_after - rc_before)
fd_deltas[i] = get_pooled_int(fd_after - fd_before)
rc_before = rc_after
fd_before = fd_after
print >> sys.stderr

if not quiet:
print >> sys.stderr

# These checkers return False on success, True on failure
def check_rc_deltas(deltas):
Expand All @@ -1513,6 +1532,7 @@ def check_fd_deltas(deltas):
(rc_deltas, 'references', check_rc_deltas),
(fd_deltas, 'file descriptors', check_fd_deltas)
]:
deltas = deltas[nwarmup:]
if checker(deltas):
msg = '%s leaked %s %s, sum=%s' % (test, deltas, item_name, sum(deltas))
print >> sys.stderr, msg
Expand Down Expand Up @@ -1647,7 +1667,7 @@ def clear_caches():
ctypes._reset_cache()

# Collect cyclic trash.
gc.collect()
support.gc_collect()

def warm_caches():
"""Create explicitly internal singletons which are created on demand
Expand Down
3 changes: 3 additions & 0 deletions Misc/NEWS.d/next/Tests/2019-04-09-14-08-02.bpo-36560._ejeOr.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Fix reference leak hunting in regrtest: compute also deltas (of reference count
and file descriptor count) during warmup, to ensure that everything is
initialized before starting to hunt reference leaks.
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy