From b11469a4d20877e07f7e11b2ee96d596692eb5d4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 1 May 2025 23:25:19 +0200 Subject: [PATCH 01/31] GH-91048: Add utils for printing the call stack for asyncio tasks --- Lib/asyncio/tools.py | 145 ++++++++++++++++++ Lib/test/test_external_inspection.py | 10 +- Lib/test/test_sys.py | 2 +- Modules/Setup | 2 +- Modules/Setup.stdlib.in | 2 +- ...linspection.c => _remotedebuggingmodule.c} | 96 ++++++++---- ...ction.vcxproj => _remotedebugging.vcxproj} | 4 +- ...lters => _remotedebugging.vcxproj.filters} | 2 +- PCbuild/pcbuild.proj | 4 +- PCbuild/pcbuild.sln | 2 +- Tools/build/generate_stdlib_module_names.py | 2 +- configure | 40 ++--- configure.ac | 4 +- 13 files changed, 244 insertions(+), 71 deletions(-) create mode 100644 Lib/asyncio/tools.py rename Modules/{_testexternalinspection.c => _remotedebuggingmodule.c} (95%) rename PCbuild/{_testexternalinspection.vcxproj => _remotedebugging.vcxproj} (97%) rename PCbuild/{_testexternalinspection.vcxproj.filters => _remotedebugging.vcxproj.filters} (90%) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py new file mode 100644 index 00000000000000..9721a1951cfdbf --- /dev/null +++ b/Lib/asyncio/tools.py @@ -0,0 +1,145 @@ +import argparse +from collections import defaultdict +from itertools import count +from enum import Enum +import sys +from _remotedebugging import get_all_awaited_by + + +class NodeType(Enum): + COROUTINE = 1 + TASK = 2 + + +# ─── indexing helpers ─────────────────────────────────────────── +def _index(result): + id2name, awaits = {}, [] + for _thr_id, tasks in result: + for tid, tname, awaited in tasks: + id2name[tid] = tname + for stack, parent_id in awaited: + awaits.append((parent_id, stack, tid)) + return id2name, awaits + + +def _build_tree(id2name, awaits): + id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} + children = defaultdict(list) + cor_names = defaultdict(dict) # (parent) -> {frame: node} + cor_id_seq = count(1) + + def _cor_node(parent_key, frame_name): + """Return an existing or new (NodeType.COROUTINE, …) node under *parent_key*.""" + bucket = cor_names[parent_key] + if frame_name in bucket: + return bucket[frame_name] + node_key = (NodeType.COROUTINE, f"c{next(cor_id_seq)}") + id2label[node_key] = frame_name + children[parent_key].append(node_key) + bucket[frame_name] = node_key + return node_key + + # touch every task so it’s present even if it awaits nobody + for tid in id2name: + children[(NodeType.TASK, tid)] + + # lay down parent ➜ …frames… ➜ child paths + for parent_id, stack, child_id in awaits: + cur = (NodeType.TASK, parent_id) + for frame in reversed(stack): # outer-most → inner-most + cur = _cor_node(cur, frame) + child_key = (NodeType.TASK, child_id) + if child_key not in children[cur]: + children[cur].append(child_key) + + return id2label, children + + +def _roots(id2label, children): + roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] + if roots: + return roots + all_children = {c for kids in children.values() for c in kids} + return [n for n in id2label if n not in all_children] + + +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): + """ + Pretty-print the async call tree produced by `get_all_async_stacks()`, + prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + """ + id2name, awaits = _index(result) + labels, children = _build_tree(id2name, awaits) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + result = [] + for r, root in enumerate(_roots(labels, children)): + result.append(render(root)) + return result + + +def build_task_table(result): + id2name, awaits = _index(result) + table = [] + for tid, tasks in result: + for task_id, task_name, awaited in tasks: + for stack, awaiter_id in awaited: + coroutine_chain = " -> ".join(stack) + awaiter_name = id2name.get(awaiter_id, "Unknown") + table.append( + [ + tid, + hex(task_id), + task_name, + coroutine_chain, + awaiter_name, + hex(awaiter_id), + ] + ) + + return table + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Show Python async tasks in a process") + parser.add_argument("pid", type=int, help="Process ID(s) to inspect.") + parser.add_argument( + "--tree", "-t", action="store_true", help="Display tasks in a tree format" + ) + args = parser.parse_args() + + try: + tasks = get_all_awaited_by(args.pid) + except RuntimeError as e: + print(f"Error retrieving tasks: {e}") + sys.exit(1) + + if args.tree: + # Print the async call tree + result = print_async_tree(tasks) + for tree in result: + print("\n".join(tree)) + else: + # Build and print the task table + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" + ) + print("-" * 135) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index aa05db972f068d..452b0174dfe911 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -13,13 +13,13 @@ PROCESS_VM_READV_SUPPORTED = False try: - from _testexternalinspection import PROCESS_VM_READV_SUPPORTED - from _testexternalinspection import get_stack_trace - from _testexternalinspection import get_async_stack_trace - from _testexternalinspection import get_all_awaited_by + from _remotedebuggingg import PROCESS_VM_READV_SUPPORTED + from _remotedebuggingg import get_stack_trace + from _remotedebuggingg import get_async_stack_trace + from _remotedebuggingg import get_all_awaited_by except ImportError: raise unittest.SkipTest( - "Test only runs when _testexternalinspection is available") + "Test only runs when _remotedebuggingmodule is available") def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 56413d00823f4a..10c3e0e9a1d2bb 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1960,7 +1960,7 @@ def _supports_remote_attaching(): PROCESS_VM_READV_SUPPORTED = False try: - from _testexternalinspection import PROCESS_VM_READV_SUPPORTED + from _remotedebuggingmodule import PROCESS_VM_READV_SUPPORTED except ImportError: pass diff --git a/Modules/Setup b/Modules/Setup index 65c22d48ba0bb7..530ce6d79b8918 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -286,7 +286,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c -#_testexternalinspection _testexternalinspection.c +#_remotedebuggingmodule _remotedebuggingmodule.c #_testsinglephase _testsinglephase.c # --- diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 33e60f37d19922..be4fb513e592e1 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -33,6 +33,7 @@ # Modules that should always be present (POSIX and Windows): @MODULE_ARRAY_TRUE@array arraymodule.c @MODULE__ASYNCIO_TRUE@_asyncio _asynciomodule.c +@MODULE__REMOTEDEBUGGING_TRUE@_remotedebugging _remotedebuggingmodule.c @MODULE__BISECT_TRUE@_bisect _bisectmodule.c @MODULE__CSV_TRUE@_csv _csv.c @MODULE__HEAPQ_TRUE@_heapq _heapqmodule.c @@ -186,7 +187,6 @@ @MODULE__TESTIMPORTMULTIPLE_TRUE@_testimportmultiple _testimportmultiple.c @MODULE__TESTMULTIPHASE_TRUE@_testmultiphase _testmultiphase.c @MODULE__TESTSINGLEPHASE_TRUE@_testsinglephase _testsinglephase.c -@MODULE__TESTEXTERNALINSPECTION_TRUE@_testexternalinspection _testexternalinspection.c @MODULE__CTYPES_TEST_TRUE@_ctypes_test _ctypes/_ctypes_test.c # Limited API template modules; must be built as shared modules. diff --git a/Modules/_testexternalinspection.c b/Modules/_remotedebuggingmodule.c similarity index 95% rename from Modules/_testexternalinspection.c rename to Modules/_remotedebuggingmodule.c index b65c5821443ebf..3cf14542ff0330 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_remotedebuggingmodule.c @@ -152,9 +152,9 @@ read_char(proc_handle_t *handle, uintptr_t address, char *result) } static int -read_int(proc_handle_t *handle, uintptr_t address, int *result) +read_sized_int(proc_handle_t *handle, uintptr_t address, void *result, size_t size) { - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(int), result); + int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, size, result); if (res < 0) { return -1; } @@ -376,11 +376,13 @@ parse_coro_chain( } Py_DECREF(name); - int gi_frame_state; - err = read_int( + int8_t gi_frame_state; + err = read_sized_int( handle, coro_address + offsets->gen_object.gi_frame_state, - &gi_frame_state); + &gi_frame_state, + sizeof(int8_t) + ); if (err) { return -1; } @@ -470,7 +472,8 @@ parse_task_awaited_by( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t task_address, - PyObject *awaited_by + PyObject *awaited_by, + int recurse_task ); @@ -480,7 +483,8 @@ parse_task( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t task_address, - PyObject *render_to + PyObject *render_to, + int recurse_task ) { char is_task; int err = read_char( @@ -508,8 +512,13 @@ parse_task( Py_DECREF(call_stack); if (is_task) { - PyObject *tn = parse_task_name( - handle, offsets, async_offsets, task_address); + PyObject *tn = NULL; + if (recurse_task) { + tn = parse_task_name( + handle, offsets, async_offsets, task_address); + } else { + tn = PyLong_FromUnsignedLong(task_address); + } if (tn == NULL) { goto err; } @@ -550,21 +559,23 @@ parse_task( goto err; } - PyObject *awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto err; - } - if (PyList_Append(result, awaited_by)) { + if (recurse_task) { + PyObject *awaited_by = PyList_New(0); + if (awaited_by == NULL) { + goto err; + } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + goto err; + } + /* we can operate on a borrowed one to simplify cleanup */ Py_DECREF(awaited_by); - goto err; - } - /* we can operate on a borrowed one to simplify cleanup */ - Py_DECREF(awaited_by); - if (parse_task_awaited_by(handle, offsets, async_offsets, - task_address, awaited_by) - ) { - goto err; + if (parse_task_awaited_by(handle, offsets, async_offsets, + task_address, awaited_by, 1) + ) { + goto err; + } } Py_DECREF(result); @@ -581,7 +592,8 @@ parse_tasks_in_set( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t set_addr, - PyObject *awaited_by + PyObject *awaited_by, + int recurse_task ) { uintptr_t set_obj; if (read_py_ptr( @@ -642,7 +654,9 @@ parse_tasks_in_set( offsets, async_offsets, key_addr, - awaited_by) + awaited_by, + recurse_task + ) ) { return -1; } @@ -666,7 +680,8 @@ parse_task_awaited_by( struct _Py_DebugOffsets* offsets, struct _Py_AsyncioModuleDebugOffsets* async_offsets, uintptr_t task_address, - PyObject *awaited_by + PyObject *awaited_by, + int recurse_task ) { uintptr_t task_ab_addr; int err = read_py_ptr( @@ -696,7 +711,9 @@ parse_task_awaited_by( offsets, async_offsets, task_address + async_offsets->asyncio_task_object.task_awaited_by, - awaited_by) + awaited_by, + recurse_task + ) ) { return -1; } @@ -715,7 +732,9 @@ parse_task_awaited_by( offsets, async_offsets, sub_task, - awaited_by) + awaited_by, + recurse_task + ) ) { return -1; } @@ -1060,15 +1079,24 @@ append_awaited_by_for_thread( return -1; } - PyObject *result_item = PyTuple_New(2); + PyObject* task_id = PyLong_FromUnsignedLong(task_addr); + if (task_id == NULL) { + Py_DECREF(tn); + Py_DECREF(current_awaited_by); + return -1; + } + + PyObject *result_item = PyTuple_New(3); if (result_item == NULL) { Py_DECREF(tn); Py_DECREF(current_awaited_by); + Py_DECREF(task_id); return -1; } - PyTuple_SET_ITEM(result_item, 0, tn); // steals ref - PyTuple_SET_ITEM(result_item, 1, current_awaited_by); // steals ref + PyTuple_SET_ITEM(result_item, 0, task_id); // steals ref + PyTuple_SET_ITEM(result_item, 1, tn); // steals ref + PyTuple_SET_ITEM(result_item, 2, current_awaited_by); // steals ref if (PyList_Append(result, result_item)) { Py_DECREF(result_item); return -1; @@ -1076,7 +1104,7 @@ append_awaited_by_for_thread( Py_DECREF(result_item); if (parse_task_awaited_by(handle, debug_offsets, async_offsets, - task_addr, current_awaited_by)) + task_addr, current_awaited_by, 0)) { return -1; } @@ -1499,7 +1527,7 @@ get_async_stack_trace(PyObject* self, PyObject* args) if (parse_task_awaited_by( handle, &local_debug_offsets, &local_async_debug, - running_task_addr, awaited_by) + running_task_addr, awaited_by, 1) ) { goto result_err; } @@ -1526,13 +1554,13 @@ static PyMethodDef methods[] = { static struct PyModuleDef module = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_testexternalinspection", + .m_name = "_remotedebuggingmodule", .m_size = -1, .m_methods = methods, }; PyMODINIT_FUNC -PyInit__testexternalinspection(void) +PyInit__remotedebugging(void) { PyObject* mod = PyModule_Create(&module); if (mod == NULL) { diff --git a/PCbuild/_testexternalinspection.vcxproj b/PCbuild/_remotedebugging.vcxproj similarity index 97% rename from PCbuild/_testexternalinspection.vcxproj rename to PCbuild/_remotedebugging.vcxproj index d5f347ecfec2c7..a16079f7c6c869 100644 --- a/PCbuild/_testexternalinspection.vcxproj +++ b/PCbuild/_remotedebugging.vcxproj @@ -68,7 +68,7 @@ {4D7C112F-3083-4D9E-9754-9341C14D9B39} - _testexternalinspection + _remotedebugging Win32Proj false @@ -93,7 +93,7 @@ <_ProjectFileVersion>10.0.30319.1 - + diff --git a/PCbuild/_testexternalinspection.vcxproj.filters b/PCbuild/_remotedebugging.vcxproj.filters similarity index 90% rename from PCbuild/_testexternalinspection.vcxproj.filters rename to PCbuild/_remotedebugging.vcxproj.filters index feb4343e5c2b8c..888e2cd478aa4e 100644 --- a/PCbuild/_testexternalinspection.vcxproj.filters +++ b/PCbuild/_remotedebugging.vcxproj.filters @@ -9,7 +9,7 @@ - + diff --git a/PCbuild/pcbuild.proj b/PCbuild/pcbuild.proj index 1bf430e03debc8..eec213d7bac612 100644 --- a/PCbuild/pcbuild.proj +++ b/PCbuild/pcbuild.proj @@ -66,7 +66,7 @@ - + @@ -79,7 +79,7 @@ - + diff --git a/PCbuild/pcbuild.sln b/PCbuild/pcbuild.sln index 803bb149c905cb..d2bfb9472b10ee 100644 --- a/PCbuild/pcbuild.sln +++ b/PCbuild/pcbuild.sln @@ -81,7 +81,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testclinic", "_testclinic. EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testinternalcapi", "_testinternalcapi.vcxproj", "{900342D7-516A-4469-B1AD-59A66E49A25F}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testexternalinspection", "_testexternalinspection.vcxproj", "{4D7C112F-3083-4D9E-9754-9341C14D9B39}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_remotedebugging", "_remotedebugging.vcxproj", "{4D7C112F-3083-4D9E-9754-9341C14D9B39}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_testimportmultiple", "_testimportmultiple.vcxproj", "{36D0C52C-DF4E-45D0-8BC7-E294C3ABC781}" EndProject diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 9873890837fa8e..8feb6c317d343e 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -34,7 +34,7 @@ '_testlimitedcapi', '_testmultiphase', '_testsinglephase', - '_testexternalinspection', + '_remotedebuggingmodule', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/configure b/configure index 7dbb35f9f45f4b..82e699c745f7b9 100755 --- a/configure +++ b/configure @@ -654,8 +654,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE -MODULE__TESTEXTERNALINSPECTION_FALSE -MODULE__TESTEXTERNALINSPECTION_TRUE +MODULE__REMOTEDEBUGGING_FALSE +MODULE__REMOTEDEBUGGING_TRUE MODULE__TESTSINGLEPHASE_FALSE MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE @@ -30684,7 +30684,7 @@ case $ac_sys_system in #( py_cv_module__ctypes_test=n/a - py_cv_module__testexternalinspection=n/a + py_cv_module__remotedebuggingmodule=n/a py_cv_module__testimportmultiple=n/a py_cv_module__testmultiphase=n/a py_cv_module__testsinglephase=n/a @@ -33449,44 +33449,44 @@ fi printf "%s\n" "$py_cv_module__testsinglephase" >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _testexternalinspection" >&5 -printf %s "checking for stdlib extension module _testexternalinspection... " >&6; } - if test "$py_cv_module__testexternalinspection" != "n/a" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _remotedebuggingmodule" >&5 +printf %s "checking for stdlib extension module _remotedebuggingmodule... " >&6; } + if test "$py_cv_module__remotedebuggingmodule" != "n/a" then : if test "$TEST_MODULES" = yes then : if true then : - py_cv_module__testexternalinspection=yes + py_cv_module__remotedebuggingmodule=yes else case e in #( - e) py_cv_module__testexternalinspection=missing ;; + e) py_cv_module__remotedebuggingmodule=missing ;; esac fi else case e in #( - e) py_cv_module__testexternalinspection=disabled ;; + e) py_cv_module__remotedebuggingmodule=disabled ;; esac fi fi - as_fn_append MODULE_BLOCK "MODULE__TESTEXTERNALINSPECTION_STATE=$py_cv_module__testexternalinspection$as_nl" - if test "x$py_cv_module__testexternalinspection" = xyes + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" + if test "x$py_cv_module__remotedebuggingmodule" = xyes then : fi - if test "$py_cv_module__testexternalinspection" = yes; then - MODULE__TESTEXTERNALINSPECTION_TRUE= - MODULE__TESTEXTERNALINSPECTION_FALSE='#' + if test "$py_cv_module__remotedebuggingmodule" = yes; then + MODULE__REMOTEDEBUGGING_TRUE= + MODULE__REMOTEDEBUGGING_FALSE='#' else - MODULE__TESTEXTERNALINSPECTION_TRUE='#' - MODULE__TESTEXTERNALINSPECTION_FALSE= + MODULE__REMOTEDEBUGGING_TRUE='#' + MODULE__REMOTEDEBUGGING_FALSE= fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__testexternalinspection" >&5 -printf "%s\n" "$py_cv_module__testexternalinspection" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 +printf "%s\n" "$py_cv_module__remotedebuggingmodule" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 @@ -34119,8 +34119,8 @@ if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHA as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__TESTEXTERNALINSPECTION_TRUE}" && test -z "${MODULE__TESTEXTERNALINSPECTION_FALSE}"; then - as_fn_error $? "conditional \"MODULE__TESTEXTERNALINSPECTION\" was never defined. +if test -z "${MODULE__REMOTEDEBUGGING_TRUE}" && test -z "${MODULE__REMOTEDEBUGGING_FALSE}"; then + as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGING\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then diff --git a/configure.ac b/configure.ac index 65f265045ba318..fa538da673f770 100644 --- a/configure.ac +++ b/configure.ac @@ -7720,7 +7720,7 @@ AS_CASE([$ac_sys_system], dnl (see Modules/Setup.stdlib.in). PY_STDLIB_MOD_SET_NA( [_ctypes_test], - [_testexternalinspection], + [_remotedebuggingmodule], [_testimportmultiple], [_testmultiphase], [_testsinglephase], @@ -8082,7 +8082,7 @@ PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([_testexternalinspection], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_remotedebuggingmodule], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], From 7f800e8f50ac6d61ba4ab400f9760d54ffa58d40 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 14:16:16 +0200 Subject: [PATCH 02/31] Maybe --- Lib/asyncio/tools.py | 105 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 95 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 9721a1951cfdbf..492a11eada0b3f 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -56,18 +56,93 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] - if roots: - return roots + """Return every task that is *not awaited by anybody*.""" all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] +# ─── helpers for _roots() ───────────────────────────────────────── +from collections import defaultdict + +def _roots(id2label, children): + """ + Return one root per *source* strongly-connected component (SCC). + + • Build the graph that contains **only tasks** as nodes and edges + parent-task ─▶ child-task (ignore coroutine frames). + + • Collapse it into SCCs with Tarjan (linear time). + + • For every component whose condensation-DAG in-degree is 0, choose a + stable representative (lexicographically-smallest label, fallback to + smallest object-id) and return that list. + """ + TASK = NodeType.TASK + task_nodes = [n for n in id2label if n[0] == TASK] + + # ------------ adjacency list among *tasks only* ----------------- + adj = defaultdict(list) + for p in task_nodes: + adj[p] = [c for c in children.get(p, []) if c[0] == TASK] + + # ------------ Tarjan’s algorithm -------------------------------- + index = 0 + stack, on_stack = [], set() + node_index, low = {}, {} + comp_of = {} # node -> comp-id + comps = defaultdict(list) # comp-id -> [members] + + def strong(v): + nonlocal index + node_index[v] = low[v] = index + index += 1 + stack.append(v) + on_stack.add(v) + + for w in adj[v]: + if w not in node_index: + strong(w) + low[v] = min(low[v], low[w]) + elif w in on_stack: + low[v] = min(low[v], node_index[w]) + + if low[v] == node_index[v]: # root of an SCC + while True: + w = stack.pop() + on_stack.remove(w) + comp_of[w] = v # use root-node as comp-id + comps[v].append(w) + if w == v: + break + + for v in task_nodes: + if v not in node_index: + strong(v) + + # ------------ condensation DAG in-degrees ----------------------- + indeg = defaultdict(int) + for p in task_nodes: + cp = comp_of[p] + for q in adj[p]: + cq = comp_of[q] + if cp != cq: + indeg[cq] += 1 + + # ------------ choose one representative per source-SCC ---------- + roots = [] + for cid, members in comps.items(): + if indeg[cid] == 0: # source component + roots.append(min( + members, + key=lambda n: (id2label[n], n[1]) # stable pick + )) + return roots + # ─── PRINT TREE FUNCTION ─────────────────────────────────────── def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ Pretty-print the async call tree produced by `get_all_async_stacks()`, - prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + coping safely with cycles. """ id2name, awaits = _index(result) labels, children = _build_tree(id2name, awaits) @@ -76,20 +151,29 @@ def pretty(node): flag = task_emoji if node[0] == NodeType.TASK else cor_emoji return f"{flag} {labels[node]}" - def render(node, prefix="", last=True, buf=None): + def render(node, prefix="", last=True, buf=None, ancestry=frozenset()): + """ + DFS renderer that stops if *node* already occurs in *ancestry* + (i.e. we just found a cycle). + """ if buf is None: buf = [] + + if node in ancestry: + buf.append(f"{prefix}{'└── ' if last else '├── '}↺ {pretty(node)} (cycle)") + return buf + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") new_pref = prefix + (" " if last else "│ ") kids = children.get(node, []) for i, kid in enumerate(kids): - render(kid, new_pref, i == len(kids) - 1, buf) + render(kid, new_pref, i == len(kids) - 1, buf, ancestry | {node}) return buf - result = [] - for r, root in enumerate(_roots(labels, children)): - result.append(render(root)) - return result + forest = [] + for root in _roots(labels, children): + forest.append(render(root)) + return forest def build_task_table(result): @@ -124,6 +208,7 @@ def build_task_table(result): try: tasks = get_all_awaited_by(args.pid) + print(tasks) except RuntimeError as e: print(f"Error retrieving tasks: {e}") sys.exit(1) From c5e4efe5d3bdb38d749e8886c80cc628712dc81e Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 14:27:57 +0200 Subject: [PATCH 03/31] Maybe --- Lib/asyncio/tools.py | 148 ++++++++++++++++++++++--------------------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 492a11eada0b3f..f25156ad339ea4 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -56,89 +56,80 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - """Return every task that is *not awaited by anybody*.""" + roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] + if roots: + return roots all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] -# ─── helpers for _roots() ───────────────────────────────────────── -from collections import defaultdict +# ─── detect cycles in the task-to-task graph ─────────────────────── +def _task_graph(awaits): + """Return {parent_task_id: {child_task_id, …}, …}.""" + from collections import defaultdict + g = defaultdict(set) + for parent_id, _stack, child_id in awaits: + g[parent_id].add(child_id) + return g -def _roots(id2label, children): + +def _find_cycles(graph): """ - Return one root per *source* strongly-connected component (SCC). + Depth-first search for back-edges. - • Build the graph that contains **only tasks** as nodes and edges - parent-task ─▶ child-task (ignore coroutine frames). + Returns a list of cycles (each cycle is a list of task-ids) or an + empty list if the graph is acyclic. + """ + WHITE, GREY, BLACK = 0, 1, 2 + color = {n: WHITE for n in graph} + path, cycles = [], [] + + def dfs(v): + color[v] = GREY + path.append(v) + for w in graph.get(v, ()): + if color[w] == WHITE: + dfs(w) + elif color[w] == GREY: # back-edge → cycle! + i = path.index(w) + cycles.append(path[i:] + [w]) # make a copy + color[v] = BLACK + path.pop() + + for v in list(graph): + if color[v] == WHITE: + dfs(v) + return cycles - • Collapse it into SCCs with Tarjan (linear time). - • For every component whose condensation-DAG in-degree is 0, choose a - stable representative (lexicographically-smallest label, fallback to - smallest object-id) and return that list. +# ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ - TASK = NodeType.TASK - task_nodes = [n for n in id2label if n[0] == TASK] - - # ------------ adjacency list among *tasks only* ----------------- - adj = defaultdict(list) - for p in task_nodes: - adj[p] = [c for c in children.get(p, []) if c[0] == TASK] - - # ------------ Tarjan’s algorithm -------------------------------- - index = 0 - stack, on_stack = [], set() - node_index, low = {}, {} - comp_of = {} # node -> comp-id - comps = defaultdict(list) # comp-id -> [members] - - def strong(v): - nonlocal index - node_index[v] = low[v] = index - index += 1 - stack.append(v) - on_stack.add(v) - - for w in adj[v]: - if w not in node_index: - strong(w) - low[v] = min(low[v], low[w]) - elif w in on_stack: - low[v] = min(low[v], node_index[w]) - - if low[v] == node_index[v]: # root of an SCC - while True: - w = stack.pop() - on_stack.remove(w) - comp_of[w] = v # use root-node as comp-id - comps[v].append(w) - if w == v: - break - - for v in task_nodes: - if v not in node_index: - strong(v) - - # ------------ condensation DAG in-degrees ----------------------- - indeg = defaultdict(int) - for p in task_nodes: - cp = comp_of[p] - for q in adj[p]: - cq = comp_of[q] - if cp != cq: - indeg[cq] += 1 - - # ------------ choose one representative per source-SCC ---------- - roots = [] - for cid, members in comps.items(): - if indeg[cid] == 0: # source component - roots.append(min( - members, - key=lambda n: (id2label[n], n[1]) # stable pick - )) - return roots + Pretty-print the async call tree produced by `get_all_async_stacks()`, + prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + """ + id2name, awaits = _index(result) + labels, children = _build_tree(id2name, awaits) + + def pretty(node): + flag = task_emoji if node[0] == NodeType.TASK else cor_emoji + return f"{flag} {labels[node]}" + + def render(node, prefix="", last=True, buf=None): + if buf is None: + buf = [] + buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") + new_pref = prefix + (" " if last else "│ ") + kids = children.get(node, []) + for i, kid in enumerate(kids): + render(kid, new_pref, i == len(kids) - 1, buf) + return buf + + result = [] + for r, root in enumerate(_roots(labels, children)): + result.append(render(root)) + return result -# ─── PRINT TREE FUNCTION ─────────────────────────────────────── def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ Pretty-print the async call tree produced by `get_all_async_stacks()`, @@ -208,13 +199,24 @@ def build_task_table(result): try: tasks = get_all_awaited_by(args.pid) - print(tasks) except RuntimeError as e: print(f"Error retrieving tasks: {e}") sys.exit(1) if args.tree: # Print the async call tree + id2name, awaits = _index(tasks) + g = _task_graph(awaits) + cycles = _find_cycles(g) + + if cycles: + print("ERROR: await-graph contains cycles – cannot print a tree!\n") + for c in cycles: + # pretty-print task names instead of bare ids + names = " → ".join(id2name.get(tid, hex(tid)) for tid in c) + print(f" cycle: {names}") + sys.exit(1) + result = print_async_tree(tasks) for tree in result: print("\n".join(tree)) From 1c982b1f98d6b2da5ad597ec80845c0c2d436c93 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 14:31:35 +0200 Subject: [PATCH 04/31] Maybe --- Lib/asyncio/tools.py | 37 ------------------------------------- 1 file changed, 37 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index f25156ad339ea4..8bf24d169dbcbb 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -130,43 +130,6 @@ def render(node, prefix="", last=True, buf=None): return result -def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): - """ - Pretty-print the async call tree produced by `get_all_async_stacks()`, - coping safely with cycles. - """ - id2name, awaits = _index(result) - labels, children = _build_tree(id2name, awaits) - - def pretty(node): - flag = task_emoji if node[0] == NodeType.TASK else cor_emoji - return f"{flag} {labels[node]}" - - def render(node, prefix="", last=True, buf=None, ancestry=frozenset()): - """ - DFS renderer that stops if *node* already occurs in *ancestry* - (i.e. we just found a cycle). - """ - if buf is None: - buf = [] - - if node in ancestry: - buf.append(f"{prefix}{'└── ' if last else '├── '}↺ {pretty(node)} (cycle)") - return buf - - buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}") - new_pref = prefix + (" " if last else "│ ") - kids = children.get(node, []) - for i, kid in enumerate(kids): - render(kid, new_pref, i == len(kids) - 1, buf, ancestry | {node}) - return buf - - forest = [] - for root in _roots(labels, children): - forest.append(render(root)) - return forest - - def build_task_table(result): id2name, awaits = _index(result) table = [] From 2d94cde8902a35cdb7ba9e827e1ba0fd9bb7e70a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 19:53:26 +0200 Subject: [PATCH 05/31] fix configure --- configure | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) mode change 100755 => 100644 configure diff --git a/configure b/configure old mode 100755 new mode 100644 index 82e699c745f7b9..2c77775099c099 --- a/configure +++ b/configure @@ -654,8 +654,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE -MODULE__REMOTEDEBUGGING_FALSE -MODULE__REMOTEDEBUGGING_TRUE +MODULE__REMOTEDEBUGGINGMODULE_FALSE +MODULE__REMOTEDEBUGGINGMODULE_TRUE MODULE__TESTSINGLEPHASE_FALSE MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE @@ -33469,7 +33469,7 @@ esac fi fi - as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGINGMODULE_STATE=$py_cv_module__remotedebuggingmodule$as_nl" if test "x$py_cv_module__remotedebuggingmodule" = xyes then : @@ -33478,11 +33478,11 @@ then : fi if test "$py_cv_module__remotedebuggingmodule" = yes; then - MODULE__REMOTEDEBUGGING_TRUE= - MODULE__REMOTEDEBUGGING_FALSE='#' + MODULE__REMOTEDEBUGGINGMODULE_TRUE= + MODULE__REMOTEDEBUGGINGMODULE_FALSE='#' else - MODULE__REMOTEDEBUGGING_TRUE='#' - MODULE__REMOTEDEBUGGING_FALSE= + MODULE__REMOTEDEBUGGINGMODULE_TRUE='#' + MODULE__REMOTEDEBUGGINGMODULE_FALSE= fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 @@ -34119,8 +34119,8 @@ if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHA as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__REMOTEDEBUGGING_TRUE}" && test -z "${MODULE__REMOTEDEBUGGING_FALSE}"; then - as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGING\" was never defined. +if test -z "${MODULE__REMOTEDEBUGGINGMODULE_TRUE}" && test -z "${MODULE__REMOTEDEBUGGINGMODULE_FALSE}"; then + as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGINGMODULE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then From 0a9a49628177f078768968ddd0985358aaf9a725 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Fri, 2 May 2025 20:02:39 +0200 Subject: [PATCH 06/31] fix configure --- Modules/Setup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/Setup b/Modules/Setup index 530ce6d79b8918..c3e0d9eb9344a9 100644 --- a/Modules/Setup +++ b/Modules/Setup @@ -286,7 +286,7 @@ PYTHONPATH=$(COREPYTHONPATH) #_testcapi _testcapimodule.c #_testimportmultiple _testimportmultiple.c #_testmultiphase _testmultiphase.c -#_remotedebuggingmodule _remotedebuggingmodule.c +#_remotedebugging _remotedebuggingmodule.c #_testsinglephase _testsinglephase.c # --- From db47ff39f47bef43d4a7206ccf1160f99271d2a7 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Fri, 2 May 2025 21:03:39 +0000 Subject: [PATCH 07/31] fix configure --- Lib/test/test_external_inspection.py | 8 ++++---- Modules/_remotedebuggingmodule.c | 2 +- Tools/build/generate_stdlib_module_names.py | 2 +- configure | 19 +++++++++---------- configure.ac | 4 ++-- 5 files changed, 17 insertions(+), 18 deletions(-) mode change 100644 => 100755 configure diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 452b0174dfe911..3535eb306e1958 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -13,10 +13,10 @@ PROCESS_VM_READV_SUPPORTED = False try: - from _remotedebuggingg import PROCESS_VM_READV_SUPPORTED - from _remotedebuggingg import get_stack_trace - from _remotedebuggingg import get_async_stack_trace - from _remotedebuggingg import get_all_awaited_by + from _remotedebugging import PROCESS_VM_READV_SUPPORTED + from _remotedebugging import get_stack_trace + from _remotedebugging import get_async_stack_trace + from _remotedebugging import get_all_awaited_by except ImportError: raise unittest.SkipTest( "Test only runs when _remotedebuggingmodule is available") diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index 3cf14542ff0330..e027ffb28d316f 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -1554,7 +1554,7 @@ static PyMethodDef methods[] = { static struct PyModuleDef module = { .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_remotedebuggingmodule", + .m_name = "_remotedebugging", .m_size = -1, .m_methods = methods, }; diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 8feb6c317d343e..761eecba96f291 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -34,7 +34,7 @@ '_testlimitedcapi', '_testmultiphase', '_testsinglephase', - '_remotedebuggingmodule', + '_remotedebugging', '_xxtestfuzz', 'idlelib.idle_test', 'test', diff --git a/configure b/configure old mode 100644 new mode 100755 index 2c77775099c099..cea1c45ae33484 --- a/configure +++ b/configure @@ -654,8 +654,8 @@ MODULE__XXTESTFUZZ_FALSE MODULE__XXTESTFUZZ_TRUE MODULE_XXSUBTYPE_FALSE MODULE_XXSUBTYPE_TRUE -MODULE__REMOTEDEBUGGINGMODULE_FALSE -MODULE__REMOTEDEBUGGINGMODULE_TRUE +MODULE__REMOTEDEBUGGING_FALSE +MODULE__REMOTEDEBUGGING_TRUE MODULE__TESTSINGLEPHASE_FALSE MODULE__TESTSINGLEPHASE_TRUE MODULE__TESTMULTIPHASE_FALSE @@ -33469,7 +33469,7 @@ esac fi fi - as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGINGMODULE_STATE=$py_cv_module__remotedebuggingmodule$as_nl" + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" if test "x$py_cv_module__remotedebuggingmodule" = xyes then : @@ -33478,11 +33478,11 @@ then : fi if test "$py_cv_module__remotedebuggingmodule" = yes; then - MODULE__REMOTEDEBUGGINGMODULE_TRUE= - MODULE__REMOTEDEBUGGINGMODULE_FALSE='#' + MODULE__REMOTEDEBUGGING_TRUE= + MODULE__REMOTEDEBUGGING_FALSE='#' else - MODULE__REMOTEDEBUGGINGMODULE_TRUE='#' - MODULE__REMOTEDEBUGGINGMODULE_FALSE= + MODULE__REMOTEDEBUGGING_TRUE='#' + MODULE__REMOTEDEBUGGING_FALSE= fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 @@ -34119,8 +34119,8 @@ if test -z "${MODULE__TESTSINGLEPHASE_TRUE}" && test -z "${MODULE__TESTSINGLEPHA as_fn_error $? "conditional \"MODULE__TESTSINGLEPHASE\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi -if test -z "${MODULE__REMOTEDEBUGGINGMODULE_TRUE}" && test -z "${MODULE__REMOTEDEBUGGINGMODULE_FALSE}"; then - as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGINGMODULE\" was never defined. +if test -z "${MODULE__REMOTEDEBUGGING_TRUE}" && test -z "${MODULE__REMOTEDEBUGGING_FALSE}"; then + as_fn_error $? "conditional \"MODULE__REMOTEDEBUGGING\" was never defined. Usually this means the macro was only invoked conditionally." "$LINENO" 5 fi if test -z "${MODULE_XXSUBTYPE_TRUE}" && test -z "${MODULE_XXSUBTYPE_FALSE}"; then @@ -35388,4 +35388,3 @@ if test "$ac_cv_header_stdatomic_h" != "yes"; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&5 printf "%s\n" "$as_me: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&6;} fi - diff --git a/configure.ac b/configure.ac index fa538da673f770..ed5c65ecbcc2be 100644 --- a/configure.ac +++ b/configure.ac @@ -7720,7 +7720,7 @@ AS_CASE([$ac_sys_system], dnl (see Modules/Setup.stdlib.in). PY_STDLIB_MOD_SET_NA( [_ctypes_test], - [_remotedebuggingmodule], + [_remotedebugging], [_testimportmultiple], [_testmultiphase], [_testsinglephase], @@ -8082,7 +8082,7 @@ PY_STDLIB_MOD([_testbuffer], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_testimportmultiple], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) -PY_STDLIB_MOD([_remotedebuggingmodule], [test "$TEST_MODULES" = yes]) +PY_STDLIB_MOD([_remotedebugging], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_ctypes_test], From 6f8bd4c79beee4a845469b28227178b1f628e061 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Fri, 2 May 2025 21:23:04 +0000 Subject: [PATCH 08/31] some tests + fixes --- Lib/asyncio/tools.py | 35 ++- Lib/test/test_asyncio/test_tools.py | 382 ++++++++++++++++++++++++++++ 2 files changed, 404 insertions(+), 13 deletions(-) create mode 100644 Lib/test/test_asyncio/test_tools.py diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 8bf24d169dbcbb..1228c787b0980e 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -11,6 +11,23 @@ class NodeType(Enum): TASK = 2 +class CycleFoundException(Exception): + """Raised when there is a cycle when drawing the call tree.""" + + def __init__(self, cycles, id2name): + super().__init__() + self.cycles = cycles + self.id2name = id2name + + def __str__(self): + for c in self.cycles: + names = " → ".join(self.id2name.get(tid, hex(tid)) for tid in c) + return ( + "ERROR: await-graph contains cycles – cannot print a tree!\n" + f"cycle: {names}" + ) + + # ─── indexing helpers ─────────────────────────────────────────── def _index(result): id2name, awaits = {}, [] @@ -80,7 +97,7 @@ def _find_cycles(graph): empty list if the graph is acyclic. """ WHITE, GREY, BLACK = 0, 1, 2 - color = {n: WHITE for n in graph} + color = defaultdict(lambda: WHITE) path, cycles = [], [] def dfs(v): @@ -108,6 +125,10 @@ def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. """ id2name, awaits = _index(result) + g = _task_graph(awaits) + cycles = _find_cycles(g) + if cycles: + raise CycleFoundException(cycles, id2name) labels, children = _build_tree(id2name, awaits) def pretty(node): @@ -168,18 +189,6 @@ def build_task_table(result): if args.tree: # Print the async call tree - id2name, awaits = _index(tasks) - g = _task_graph(awaits) - cycles = _find_cycles(g) - - if cycles: - print("ERROR: await-graph contains cycles – cannot print a tree!\n") - for c in cycles: - # pretty-print task names instead of bare ids - names = " → ".join(id2name.get(tid, hex(tid)) for tid in c) - print(f" cycle: {names}") - sys.exit(1) - result = print_async_tree(tasks) for tree in result: print("\n".join(tree)) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py new file mode 100644 index 00000000000000..149c6db6d5ba71 --- /dev/null +++ b/Lib/test/test_asyncio/test_tools.py @@ -0,0 +1,382 @@ +"""Tests for the asyncio tools script.""" + +from Lib.asyncio import tools +from test.test_asyncio import utils as test_utils + + +# mock output of get_all_awaited_by function. +TEST_INPUTS = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "timer", + [ + [["awaiter3", "awaiter2", "awaiter"], 4], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 5], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 6], + [["awaiter3", "awaiter2", "awaiter"], 7], + ], + ), + ( + 8, + "root1", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 9, + "root2", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 4, + "child1_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 6, + "child2_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 7, + "child1_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ( + 5, + "child2_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ], + ), + (0, []), + ), + ([]), + ( + [ + [ + "└── (T) Task-1", + " └── main", + " └── __aexit__", + " └── _aexit", + " ├── (T) root1", + " │ └── bloch", + " │ └── blocho_caller", + " │ └── __aexit__", + " │ └── _aexit", + " │ ├── (T) child1_1", + " │ │ └── awaiter", + " │ │ └── awaiter2", + " │ │ └── awaiter3", + " │ │ └── (T) timer", + " │ └── (T) child2_1", + " │ └── awaiter1", + " │ └── awaiter1_2", + " │ └── awaiter1_3", + " │ └── (T) timer", + " └── (T) root2", + " └── bloch", + " └── blocho_caller", + " └── __aexit__", + " └── _aexit", + " ├── (T) child1_2", + " │ └── awaiter", + " │ └── awaiter2", + " │ └── awaiter3", + " │ └── (T) timer", + " └── (T) child2_2", + " └── awaiter1", + " └── awaiter1_2", + " └── awaiter1_3", + " └── (T) timer", + ] + ] + ), + ( + [ + [ + 1, + "0x3", + "timer", + "awaiter3 -> awaiter2 -> awaiter", + "child1_1", + "0x4", + ], + [ + 1, + "0x3", + "timer", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_2", + "0x5", + ], + [ + 1, + "0x3", + "timer", + "awaiter1_3 -> awaiter1_2 -> awaiter1", + "child2_1", + "0x6", + ], + [ + 1, + "0x3", + "timer", + "awaiter3 -> awaiter2 -> awaiter", + "child1_2", + "0x7", + ], + [ + 1, + "0x8", + "root1", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x9", + "root2", + "_aexit -> __aexit__ -> main", + "Task-1", + "0x2", + ], + [ + 1, + "0x4", + "child1_1", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x6", + "child2_1", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root1", + "0x8", + ], + [ + 1, + "0x7", + "child1_2", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + [ + 1, + "0x5", + "child2_2", + "_aexit -> __aexit__ -> blocho_caller -> bloch", + "root2", + "0x9", + ], + ] + ), + ], + [ + # test case containing two roots + ( + ( + 1, + [ + (2, "Task-1", []), + (3, "timer", [[["awaiter"], 4]]), + (4, "subtask1", [[["main"], 2]]), + ], + ), + ( + 5, + [ + (6, "Task-1", []), + (7, "sleeper", [[["awaiter2"], 8]]), + (8, "subtask2", [[["main"], 6]]), + ], + ), + (0, []), + ), + ([]), + ( + [ + [ + "└── (T) Task-1", + " └── main", + " └── (T) subtask1", + " └── awaiter", + " └── (T) timer", + ], + [ + "└── (T) Task-1", + " └── main", + " └── (T) subtask2", + " └── awaiter2", + " └── (T) sleeper", + ], + ] + ), + ( + [ + [1, "0x3", "timer", "awaiter", "subtask1", "0x4"], + [1, "0x4", "subtask1", "main", "Task-1", "0x2"], + [5, "0x7", "sleeper", "awaiter2", "subtask2", "0x8"], + [5, "0x8", "subtask2", "main", "Task-1", "0x6"], + ] + ), + ], + # Tests cycle detection. + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "a", + [[["awaiter2"], 4], [["main"], 2]], + ), + (4, "b", [[["awaiter"], 3]]), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4]]), + ([]), + ( + [ + [1, "0x3", "a", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "main", "Task-1", "0x2"], + [1, "0x4", "b", "awaiter", "a", "0x3"], + ] + ), + ], + [ + # this test case contains two cycles + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "A", + [[["nested", "nested", "task_b"], 4]], + ), + ( + 4, + "B", + [ + [["nested", "nested", "task_c"], 5], + [["nested", "nested", "task_a"], 3], + ], + ), + (5, "C", [[["nested", "nested"], 6]]), + ( + 6, + "Task-2", + [[["nested", "nested", "task_b"], 4]], + ), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4], [4, 6, 5, 4]]), + ([]), + ( + [ + [ + 1, + "0x3", + "A", + "nested -> nested -> task_b", + "B", + "0x4", + ], + [ + 1, + "0x4", + "B", + "nested -> nested -> task_c", + "C", + "0x5", + ], + [ + 1, + "0x4", + "B", + "nested -> nested -> task_a", + "A", + "0x3", + ], + [ + 1, + "0x5", + "C", + "nested -> nested", + "Task-2", + "0x6", + ], + [ + 1, + "0x6", + "Task-2", + "nested -> nested -> task_b", + "B", + "0x4", + ], + ] + ), + ], +] + + +class TestAsyncioTools(test_utils.TestCase): + + def test_asyncio_utils(self): + for input_, cycles, tree, table in TEST_INPUTS: + if cycles: + try: + tools.print_async_tree(input_) + except tools.CycleFoundException as e: + self.assertEqual(e.cycles, cycles) + else: + print(tools.print_async_tree(input_)) + self.assertEqual(tools.print_async_tree(input_), tree) + print(tools.build_task_table(input_)) + self.assertEqual(tools.build_task_table(input_), table) From 152b3d703c1a939a144c3dc601a556d67a9a2001 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Fri, 2 May 2025 23:39:05 +0000 Subject: [PATCH 09/31] improve tests --- Lib/asyncio/tools.py | 7 ++-- Lib/test/test_asyncio/test_tools.py | 52 ++++++++++++++++------------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 1228c787b0980e..ccfd4893ac43ab 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -73,9 +73,9 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] - if roots: - return roots + # roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] + # if roots: + # return roots all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] @@ -187,6 +187,7 @@ def build_task_table(result): print(f"Error retrieving tasks: {e}") sys.exit(1) + print(tasks) if args.tree: # Print the async call tree result = print_async_tree(tasks) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 149c6db6d5ba71..61ce4d8eb01785 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -1,7 +1,8 @@ """Tests for the asyncio tools script.""" +import unittest + from Lib.asyncio import tools -from test.test_asyncio import utils as test_utils # mock output of get_all_awaited_by function. @@ -209,48 +210,53 @@ # test case containing two roots ( ( - 1, + 9, [ - (2, "Task-1", []), - (3, "timer", [[["awaiter"], 4]]), - (4, "subtask1", [[["main"], 2]]), + (5, "Task-5", []), + (6, "Task-6", [[["main2"], 5]]), + (7, "Task-7", [[["main2"], 5]]), + (8, "Task-8", [[["main2"], 5]]), ], ), ( - 5, + 10, [ - (6, "Task-1", []), - (7, "sleeper", [[["awaiter2"], 8]]), - (8, "subtask2", [[["main"], 6]]), + (1, "Task-1", []), + (2, "Task-2", [[["main"], 1]]), + (3, "Task-3", [[["main"], 1]]), + (4, "Task-4", [[["main"], 1]]), ], ), + (11, []), (0, []), ), ([]), ( [ [ - "└── (T) Task-1", - " └── main", - " └── (T) subtask1", - " └── awaiter", - " └── (T) timer", + "└── (T) Task-5", + " └── main2", + " ├── (T) Task-6", + " ├── (T) Task-7", + " └── (T) Task-8", ], [ "└── (T) Task-1", " └── main", - " └── (T) subtask2", - " └── awaiter2", - " └── (T) sleeper", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", ], ] ), ( [ - [1, "0x3", "timer", "awaiter", "subtask1", "0x4"], - [1, "0x4", "subtask1", "main", "Task-1", "0x2"], - [5, "0x7", "sleeper", "awaiter2", "subtask2", "0x8"], - [5, "0x8", "subtask2", "main", "Task-1", "0x6"], + [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "main2", "Task-5", "0x5"], + [10, "0x2", "Task-2", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "main", "Task-1", "0x1"], ] ), ], @@ -366,7 +372,7 @@ ] -class TestAsyncioTools(test_utils.TestCase): +class TestAsyncioTools(unittest.TestCase): def test_asyncio_utils(self): for input_, cycles, tree, table in TEST_INPUTS: @@ -376,7 +382,5 @@ def test_asyncio_utils(self): except tools.CycleFoundException as e: self.assertEqual(e.cycles, cycles) else: - print(tools.print_async_tree(input_)) self.assertEqual(tools.print_async_tree(input_), tree) - print(tools.build_task_table(input_)) self.assertEqual(tools.build_task_table(input_), table) From 955ef27415d3276b852e9b4d79124bc68144cf94 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 01:45:16 +0200 Subject: [PATCH 10/31] dsf --- configure | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/configure b/configure index cea1c45ae33484..3b74554d5a2e64 100755 --- a/configure +++ b/configure @@ -30684,7 +30684,7 @@ case $ac_sys_system in #( py_cv_module__ctypes_test=n/a - py_cv_module__remotedebuggingmodule=n/a + py_cv_module__remotedebugging=n/a py_cv_module__testimportmultiple=n/a py_cv_module__testmultiphase=n/a py_cv_module__testsinglephase=n/a @@ -33449,35 +33449,35 @@ fi printf "%s\n" "$py_cv_module__testsinglephase" >&6; } - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _remotedebuggingmodule" >&5 -printf %s "checking for stdlib extension module _remotedebuggingmodule... " >&6; } - if test "$py_cv_module__remotedebuggingmodule" != "n/a" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module _remotedebugging" >&5 +printf %s "checking for stdlib extension module _remotedebugging... " >&6; } + if test "$py_cv_module__remotedebugging" != "n/a" then : if test "$TEST_MODULES" = yes then : if true then : - py_cv_module__remotedebuggingmodule=yes + py_cv_module__remotedebugging=yes else case e in #( - e) py_cv_module__remotedebuggingmodule=missing ;; + e) py_cv_module__remotedebugging=missing ;; esac fi else case e in #( - e) py_cv_module__remotedebuggingmodule=disabled ;; + e) py_cv_module__remotedebugging=disabled ;; esac fi fi - as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebuggingmodule$as_nl" - if test "x$py_cv_module__remotedebuggingmodule" = xyes + as_fn_append MODULE_BLOCK "MODULE__REMOTEDEBUGGING_STATE=$py_cv_module__remotedebugging$as_nl" + if test "x$py_cv_module__remotedebugging" = xyes then : fi - if test "$py_cv_module__remotedebuggingmodule" = yes; then + if test "$py_cv_module__remotedebugging" = yes; then MODULE__REMOTEDEBUGGING_TRUE= MODULE__REMOTEDEBUGGING_FALSE='#' else @@ -33485,8 +33485,8 @@ else MODULE__REMOTEDEBUGGING_FALSE= fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebuggingmodule" >&5 -printf "%s\n" "$py_cv_module__remotedebuggingmodule" >&6; } + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $py_cv_module__remotedebugging" >&5 +printf "%s\n" "$py_cv_module__remotedebugging" >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module xxsubtype" >&5 @@ -35388,3 +35388,4 @@ if test "$ac_cv_header_stdatomic_h" != "yes"; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&5 printf "%s\n" "$as_me: Your compiler or platform does have a working C11 stdatomic.h. A future version of Python may require stdatomic.h." >&6;} fi + From 65aee3cfe449178d8aed96755396718881b2e27c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 01:59:55 +0200 Subject: [PATCH 11/31] dsf --- Lib/test/test_external_inspection.py | 31 +++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 3535eb306e1958..e7f3e922f75a45 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -4,6 +4,7 @@ import importlib import sys import socket +from unittest.mock import ANY from test.support import os_helper, SHORT_TIMEOUT, busy_retry from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -184,13 +185,13 @@ def new_eager_loop(): root_task = "Task-1" expected_stack_trace = [ - ["c5", "c4", "c3", "c2"], - "c2_root", + ['c5', 'c4', 'c3', 'c2'], + 'c2_root', [ - [["main"], root_task, []], - [["c1"], "sub_main_1", [[["main"], root_task, []]]], - [["c1"], "sub_main_2", [[["main"], root_task, []]]], - ], + [['_aexit', '__aexit__', 'main'], root_task, []], + [['c1'], 'sub_main_1', [[['_aexit', '__aexit__', 'main'], root_task, []]]], + [['c1'], 'sub_main_2', [[['_aexit', '__aexit__', 'main'], root_task, []]]], + ] ] self.assertEqual(stack_trace, expected_stack_trace) @@ -397,8 +398,10 @@ async def main(): # sets are unordered, so we want to sort "awaited_by"s stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [ - ['deep', 'c1', 'run_one_coro'], 'Task-2', [[['main'], 'Task-1', []]] + expected_stack_trace = [ + ['deep', 'c1', 'run_one_coro'], + 'Task-2', + [[['staggered_race', 'main'], 'Task-1', []]] ] self.assertEqual(stack_trace, expected_stack_trace) @@ -516,19 +519,19 @@ async def main(): # expected: at least 1000 pending tasks self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure - self.assertIn(('Task-1', []), entries) - self.assertIn(('server task', [[['main'], 'Task-1', []]]), entries) - self.assertIn(('echo client spam', [[['main'], 'Task-1', []]]), entries) + self.assertIn((ANY, 'Task-1', []), entries) + self.assertIn((ANY, 'server task', [[['_aexit', '__aexit__', 'main'], ANY]]), entries) + self.assertIn((ANY, 'echo client spam', [[['_aexit', '__aexit__', 'main'], ANY]]), entries) - expected_stack = [[['echo_client_spam'], 'echo client spam', [[['main'], 'Task-1', []]]]] - tasks_with_stack = [task for task in entries if task[1] == expected_stack] + expected_stack = [[['_aexit', '__aexit__', 'echo_client_spam'], ANY]] + tasks_with_stack = [task for task in entries if task[2] == expected_stack] self.assertGreaterEqual(len(tasks_with_stack), 1000) # the final task will have some random number, but it should for # sure be one of the echo client spam horde (In windows this is not true # for some reason) if sys.platform != "win32": - self.assertEqual([[['echo_client_spam'], 'echo client spam', [[['main'], 'Task-1', []]]]], entries[-1][1]) + self.assertEqual([[['_aexit', '__aexit__', 'echo_client_spam'], ANY]], entries[-1][2]) except PermissionError: self.skipTest( "Insufficient permissions to read the stack trace") From 51e689ed04fd38afb7e637ff88b564eaefd76732 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 02:12:15 +0200 Subject: [PATCH 12/31] test fixes --- Lib/test/test_asyncio/test_tools.py | 227 ++++++++++++++++++++++++---- 1 file changed, 195 insertions(+), 32 deletions(-) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 61ce4d8eb01785..59cdca4ce6bfec 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -6,7 +6,7 @@ # mock output of get_all_awaited_by function. -TEST_INPUTS = [ +TEST_INPUTS_TREE = [ [ # test case containing a task called timer being awaited in two # different subtasks part of a TaskGroup (root1 and root2) which call @@ -80,7 +80,6 @@ ), (0, []), ), - ([]), ( [ [ @@ -121,6 +120,184 @@ ] ] ), + ], + [ + # test case containing two roots + ( + ( + 9, + [ + (5, "Task-5", []), + (6, "Task-6", [[["main2"], 5]]), + (7, "Task-7", [[["main2"], 5]]), + (8, "Task-8", [[["main2"], 5]]), + ], + ), + ( + 10, + [ + (1, "Task-1", []), + (2, "Task-2", [[["main"], 1]]), + (3, "Task-3", [[["main"], 1]]), + (4, "Task-4", [[["main"], 1]]), + ], + ), + (11, []), + (0, []), + ), + ( + [ + [ + "└── (T) Task-5", + " └── main2", + " ├── (T) Task-6", + " ├── (T) Task-7", + " └── (T) Task-8", + ], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], +] + +TEST_INPUTS_CYCLES_TREE = [ + [ + # this test case contains a cycle: two tasks awaiting each other. + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "a", + [[["awaiter2"], 4], [["main"], 2]], + ), + (4, "b", [[["awaiter"], 3]]), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4]]), + ], + [ + # this test case contains two cycles + ( + [ + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "A", + [[["nested", "nested", "task_b"], 4]], + ), + ( + 4, + "B", + [ + [["nested", "nested", "task_c"], 5], + [["nested", "nested", "task_a"], 3], + ], + ), + (5, "C", [[["nested", "nested"], 6]]), + ( + 6, + "Task-2", + [[["nested", "nested", "task_b"], 4]], + ), + ], + ), + (0, []), + ] + ), + ([[4, 3, 4], [4, 6, 5, 4]]), + ], +] + +TEST_INPUTS_TABLE = [ + [ + # test case containing a task called timer being awaited in two + # different subtasks part of a TaskGroup (root1 and root2) which call + # awaiter functions. + ( + ( + 1, + [ + (2, "Task-1", []), + ( + 3, + "timer", + [ + [["awaiter3", "awaiter2", "awaiter"], 4], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 5], + [["awaiter1_3", "awaiter1_2", "awaiter1"], 6], + [["awaiter3", "awaiter2", "awaiter"], 7], + ], + ), + ( + 8, + "root1", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 9, + "root2", + [[["_aexit", "__aexit__", "main"], 2]], + ), + ( + 4, + "child1_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 6, + "child2_1", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 8, + ] + ], + ), + ( + 7, + "child1_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ( + 5, + "child2_2", + [ + [ + ["_aexit", "__aexit__", "blocho_caller", "bloch"], + 9, + ] + ], + ), + ], + ), + (0, []), + ), ( [ [ @@ -230,25 +407,6 @@ (11, []), (0, []), ), - ([]), - ( - [ - [ - "└── (T) Task-5", - " └── main2", - " ├── (T) Task-6", - " ├── (T) Task-7", - " └── (T) Task-8", - ], - [ - "└── (T) Task-1", - " └── main", - " ├── (T) Task-2", - " ├── (T) Task-3", - " └── (T) Task-4", - ], - ] - ), ( [ [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], @@ -260,7 +418,7 @@ ] ), ], - # Tests cycle detection. + # CASES WITH CYCLES [ # this test case contains a cycle: two tasks awaiting each other. ( @@ -280,8 +438,6 @@ (0, []), ] ), - ([[4, 3, 4]]), - ([]), ( [ [1, "0x3", "a", "awaiter2", "b", "0x4"], @@ -322,8 +478,6 @@ (0, []), ] ), - ([[4, 3, 4], [4, 6, 5, 4]]), - ([]), ( [ [ @@ -372,15 +526,24 @@ ] -class TestAsyncioTools(unittest.TestCase): +class TestAsyncioToolsTree(unittest.TestCase): def test_asyncio_utils(self): - for input_, cycles, tree, table in TEST_INPUTS: - if cycles: + for input_, tree in TEST_INPUTS_TREE: + with self.subTest(input_): + self.assertEqual(tools.print_async_tree(input_), tree) + + def test_asyncio_utils_cycles(self): + for input_, cycles in TEST_INPUTS_CYCLES_TREE: + with self.subTest(input_): try: tools.print_async_tree(input_) except tools.CycleFoundException as e: self.assertEqual(e.cycles, cycles) - else: - self.assertEqual(tools.print_async_tree(input_), tree) - self.assertEqual(tools.build_task_table(input_), table) + + +class TestAsyncioToolsTable(unittest.TestCase): + def test_asyncio_utils(self): + for input_, table in TEST_INPUTS_TABLE: + with self.subTest(input_): + self.assertEqual(tools.build_task_table(input_), table) From 1d2734864a914e5e87b4cac665b7ed099103e212 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 02:25:01 +0200 Subject: [PATCH 13/31] test fixes --- Lib/asyncio/tools.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index ccfd4893ac43ab..d966c35c1f5135 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,4 +1,5 @@ import argparse +from dataclasses import dataclass from collections import defaultdict from itertools import count from enum import Enum @@ -11,21 +12,11 @@ class NodeType(Enum): TASK = 2 +@dataclass(frozen=True) class CycleFoundException(Exception): """Raised when there is a cycle when drawing the call tree.""" - - def __init__(self, cycles, id2name): - super().__init__() - self.cycles = cycles - self.id2name = id2name - - def __str__(self): - for c in self.cycles: - names = " → ".join(self.id2name.get(tid, hex(tid)) for tid in c) - return ( - "ERROR: await-graph contains cycles – cannot print a tree!\n" - f"cycle: {names}" - ) + cycles: list[list[int]] + id2name: dict[int, str] # ─── indexing helpers ─────────────────────────────────────────── @@ -172,6 +163,14 @@ def build_task_table(result): return table +def _print_cycle_exception(exception: CycleFoundException): + print("ERROR: await-graph contains cycles – cannot print a tree!", file=sys.stderr) + print("", file=sys.stderr) + for c in exception.cycles: + inames = " → ".join(exception.id2name.get(tid, hex(tid)) for tid in c) + print(f"cycle: {inames}", file=sys.stderr) + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Show Python async tasks in a process") @@ -184,13 +183,19 @@ def build_task_table(result): try: tasks = get_all_awaited_by(args.pid) except RuntimeError as e: + while e.__cause__ is not None: + e = e.__cause__ print(f"Error retrieving tasks: {e}") sys.exit(1) - print(tasks) if args.tree: # Print the async call tree - result = print_async_tree(tasks) + try: + result = print_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + for tree in result: print("\n".join(tree)) else: From 1d1b0e9b2ebf739ea7be5886c390e46624205725 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 02:27:35 +0200 Subject: [PATCH 14/31] test fixes --- Lib/asyncio/tools.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index d966c35c1f5135..1719821a1ffe6e 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -64,9 +64,6 @@ def _cor_node(parent_key, frame_name): def _roots(id2label, children): - # roots = [n for n, lbl in id2label.items() if lbl == "Task-1"] - # if roots: - # return roots all_children = {c for kids in children.values() for c in kids} return [n for n in id2label if n not in all_children] @@ -183,8 +180,8 @@ def _print_cycle_exception(exception: CycleFoundException): try: tasks = get_all_awaited_by(args.pid) except RuntimeError as e: - while e.__cause__ is not None: - e = e.__cause__ + while e.__context__ is not None: + e = e.__context__ print(f"Error retrieving tasks: {e}") sys.exit(1) From edad4d1f983f3b075188b438e25fa0ee21416806 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 03:18:48 +0200 Subject: [PATCH 15/31] test fixes --- Lib/test/test_asyncio/test_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 59cdca4ce6bfec..6ad47ef9e874bf 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -2,7 +2,7 @@ import unittest -from Lib.asyncio import tools +from asyncio import tools # mock output of get_all_awaited_by function. From 199589ceb14d1e72202bb58032e5a037d842c8b7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 13:19:16 +0200 Subject: [PATCH 16/31] Fix free threading offsets --- Modules/_remotedebuggingmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index e027ffb28d316f..84d97858fe2ed7 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -345,7 +345,7 @@ parse_coro_chain( uintptr_t gen_type_addr; int err = read_ptr( handle, - coro_address + sizeof(void*), + coro_address + offsets->pyobject.ob_type, &gen_type_addr); if (err) { return -1; From 9e87032d62c6f2c6c0bbd5f36caefd16ac969902 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 13:32:08 +0200 Subject: [PATCH 17/31] Fix free threading offsets AGAIN --- Modules/_remotedebuggingmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index 84d97858fe2ed7..f6e8135a6c5800 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -429,7 +429,7 @@ parse_coro_chain( uintptr_t gi_await_addr_type_addr; int err = read_ptr( handle, - gi_await_addr + sizeof(void*), + gi_await_addr + offsets->pyobject.ob_type, &gi_await_addr_type_addr); if (err) { return -1; From 69e9221f54965da54bcfe40c82929f9705ad6ef4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 18:36:18 +0200 Subject: [PATCH 18/31] Debugging --- Lib/test/test_external_inspection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index e7f3e922f75a45..0c8fc0395b65cc 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -517,6 +517,7 @@ async def main(): self.assertEqual(all_awaited_by[1], (0, [])) entries = all_awaited_by[0][1] # expected: at least 1000 pending tasks + print(entries, file=sys.stderr) self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure self.assertIn((ANY, 'Task-1', []), entries) From b6cb609812f6ce27fcc1552c05e248a6b6f1e591 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 18:54:44 +0200 Subject: [PATCH 19/31] More tests --- Lib/test/test_asyncio/test_tools.py | 224 ++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 6ad47ef9e874bf..2f2b5feac43d74 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -547,3 +547,227 @@ def test_asyncio_utils(self): for input_, table in TEST_INPUTS_TABLE: with self.subTest(input_): self.assertEqual(tools.build_task_table(input_), table) + + +class TestAsyncioToolsBasic(unittest.TestCase): + def test_empty_input_tree(self): + """Test print_async_tree with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.print_async_tree(result), expected_output) + + def test_empty_input_table(self): + """Test build_task_table with empty input.""" + result = [] + expected_output = [] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_only_independent_tasks_tree(self): + input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + expected = [["└── (T) taskA"], ["└── (T) taskB"]] + result = tools.print_async_tree(input_) + self.assertEqual(sorted(result), sorted(expected)) + + def test_only_independent_tasks_table(self): + input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + self.assertEqual(tools.build_task_table(input_), []) + + def test_single_task_tree(self): + """Test print_async_tree with a single task and no awaits.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + ], + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + ] + ] + self.assertEqual(tools.print_async_tree(result), expected_output) + + def test_single_task_table(self): + """Test build_task_table with a single task and no awaits.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + ], + ) + ] + expected_output = [] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_cycle_detection(self): + """Test print_async_tree raises CycleFoundException for cyclic input.""" + result = [ + ( + 1, + [ + (2, "Task-1", [[["main"], 3]]), + (3, "Task-2", [[["main"], 2]]), + ], + ) + ] + with self.assertRaises(tools.CycleFoundException) as context: + tools.print_async_tree(result) + self.assertEqual(context.exception.cycles, [[3, 2, 3]]) + + def test_complex_tree(self): + """Test print_async_tree with a more complex tree structure.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + (3, "Task-2", [[["main"], 2]]), + (4, "Task-3", [[["main"], 3]]), + ], + ) + ] + expected_output = [ + [ + "└── (T) Task-1", + " └── main", + " └── (T) Task-2", + " └── main", + " └── (T) Task-3", + ] + ] + self.assertEqual(tools.print_async_tree(result), expected_output) + + def test_complex_table(self): + """Test build_task_table with a more complex tree structure.""" + result = [ + ( + 1, + [ + (2, "Task-1", []), + (3, "Task-2", [[["main"], 2]]), + (4, "Task-3", [[["main"], 3]]), + ], + ) + ] + expected_output = [ + [1, "0x3", "Task-2", "main", "Task-1", "0x2"], + [1, "0x4", "Task-3", "main", "Task-2", "0x3"], + ] + self.assertEqual(tools.build_task_table(result), expected_output) + + def test_deep_coroutine_chain(self): + input_ = [ + ( + 1, + [ + (10, "leaf", [[["c1", "c2", "c3", "c4", "c5"], 11]]), + (11, "root", []), + ], + ) + ] + expected = [ + [ + "└── (T) root", + " └── c5", + " └── c4", + " └── c3", + " └── c2", + " └── c1", + " └── (T) leaf", + ] + ] + result = tools.print_async_tree(input_) + self.assertEqual(result, expected) + + def test_multiple_cycles_same_node(self): + input_ = [ + ( + 1, + [ + (1, "Task-A", [[["call1"], 2]]), + (2, "Task-B", [[["call2"], 3]]), + (3, "Task-C", [[["call3"], 1], [["call4"], 2]]), + ], + ) + ] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.print_async_tree(input_) + cycles = ctx.exception.cycles + self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) + + def test_table_output_format(self): + input_ = [(1, [(1, "Task-A", [[["foo"], 2]]), (2, "Task-B", [])])] + table = tools.build_task_table(input_) + for row in table: + self.assertEqual(len(row), 6) + self.assertIsInstance(row[0], int) # thread ID + self.assertTrue( + isinstance(row[1], str) and row[1].startswith("0x") + ) # hex task ID + self.assertIsInstance(row[2], str) # task name + self.assertIsInstance(row[3], str) # coroutine chain + self.assertIsInstance(row[4], str) # awaiter name + self.assertTrue( + isinstance(row[5], str) and row[5].startswith("0x") + ) # hex awaiter ID + + +class TestAsyncioToolsEdgeCases(unittest.TestCase): + + def test_task_awaits_self(self): + """A task directly awaits itself – should raise a cycle.""" + input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])] + with self.assertRaises(tools.CycleFoundException) as ctx: + tools.print_async_tree(input_) + self.assertIn([1, 1], ctx.exception.cycles) + + def test_task_with_missing_awaiter_id(self): + """Awaiter ID not in task list – should not crash, just show 'Unknown'.""" + input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])] # 999 not defined + table = tools.build_task_table(input_) + self.assertEqual(len(table), 1) + self.assertEqual(table[0][4], "Unknown") + + def test_duplicate_coroutine_frames(self): + """Same coroutine frame repeated under a parent – should deduplicate.""" + input_ = [ + ( + 1, + [ + (1, "Task-1", [[["frameA"], 2], [["frameA"], 3]]), + (2, "Task-2", []), + (3, "Task-3", []), + ], + ) + ] + tree = tools.print_async_tree(input_) + # Both children should be under the same coroutine node + flat = "\n".join(tree[0]) + self.assertIn("frameA", flat) + self.assertIn("Task-2", flat) + self.assertIn("Task-1", flat) + + flat = "\n".join(tree[1]) + self.assertIn("frameA", flat) + self.assertIn("Task-3", flat) + self.assertIn("Task-1", flat) + + def test_task_with_no_name(self): + """Task with no name in id2name – should still render with fallback.""" + input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])] + # If name is None, fallback to string should not crash + tree = tools.print_async_tree(input_) + self.assertIn("(T) None", "\n".join(tree[0])) + + def test_tree_rendering_with_custom_emojis(self): + """Pass custom emojis to the tree renderer.""" + input_ = [(1, [(1, "MainTask", [[["f1", "f2"], 2]]), (2, "SubTask", [])])] + tree = tools.print_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") + flat = "\n".join(tree[0]) + self.assertIn("🧵 MainTask", flat) + self.assertIn("🔁 f1", flat) + self.assertIn("🔁 f2", flat) + self.assertIn("🧵 SubTask", flat) From 2dd34521a9429903246a04441c63ddcf55fb64ba Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 19:04:29 +0200 Subject: [PATCH 20/31] Add news entry --- Doc/whatsnew/3.14.rst | 90 +++++++++++++++++++ ...5-05-03-19-04-03.gh-issue-91048.S8QWSw.rst | 4 + 2 files changed, 94 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2f8b652d47e428..dca311880f9826 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -508,6 +508,96 @@ configuration mechanisms). .. seealso:: :pep:`741`. +.. _whatsnew314-asyncio-introspection + +Asyncio introspection capabilities +---------------------------------- + +Added a new command-line interface to inspect running Python processes using +asynchronous tasks, available via: + +.. code-block:: bash + python -m asyncio.tools [--tree] PID + +This tool inspects the given process ID (PID) and displays information about +currently running asyncio tasks. By default, it outputs a task table: a flat +listing of all tasks, their names, their coroutine stacks, and which tasks are +awaiting them. + +With the ``--tree`` option, it instead renders a visual async call tree, +showing coroutine relationships in a hierarchical format. This command is +particularly useful for debugging long-running or stuck asynchronous programs. +It can help developers quickly identify where a program is blocked, what tasks +are pending, and how coroutines are chained together. + +For example given this code: + +.. code-block:: python + + import asyncio + + async def sleeper(name, delay): + await asyncio.sleep(delay) + print(f"{name} is done sleeping.") + + async def inner(name): + await asyncio.sleep(0.1) + await sleeper(name, 1) + + async def task_group(name): + await asyncio.gather( + inner(f"{name}-1"), + inner(f"{name}-2"), + ) + + async def main(): + # Start two separate task groups + t1 = asyncio.create_task(task_group("groupA")) + t2 = asyncio.create_task(task_group("groupB")) + await t1 + await t2 + + if __name__ == "__main__": + asyncio.run(main()) + +Executing the new tool on the running process will yield a table like this: + +.. code-block:: bash + + python -m asyncio.tools 12345 + + tid task id task name coroutine chain awaiter name awaiter id + --------------------------------------------------------------------------------------------------------------------------------------- + 6826911 0x200013c0220 Task-2 main Task-1 0x200013b0020 + 6826911 0x200013c0620 Task-4 task_group Task-2 0x200013c0220 + 6826911 0x200013c0820 Task-5 task_group Task-2 0x200013c0220 + 6826911 0x200013c0c20 Task-6 task_group Task-3 0x200013c0420 + 6826911 0x200013c0e20 Task-7 task_group Task-3 0x200013c0420 + + +and with the ``--tree`` option: + +.. code-block:: bash + + python -m asyncio.tools --tree 12345 + + └── (T) Task-1 + └── main + └── (T) Task-2 + └── task_group + ├── (T) Task-4 + └── (T) Task-5 + └── (T) Task-3 + └── task_group + ├── (T) Task-6 + └── (T) Task-7 + +If a cycle is detected in the async await graph (which could indicate a +programming issue), the tool raises an error and lists the cycle paths that +prevent tree construction. + +(Contributed by Pablo Galindo, Łukasz Langa and Marta Gomez Macias in :gh:`91048`.) + .. _whatsnew314-tail-call: A new type of interpreter diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst new file mode 100644 index 00000000000000..c33810614d4d56 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst @@ -0,0 +1,4 @@ +Add a new ``python -m asyncio.tools`` command-line interface to inspect +asyncio tasks in a running Python process. Displays a flat table of await +relationships or a tree view with ``--tree``, useful for debugging async +code. Patch by Pablo Galindo, Łukasz Langa and Marta Gomez Macias. From a84a171054042c4d678d1001019c50cc544b9ca4 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 3 May 2025 19:08:24 +0200 Subject: [PATCH 21/31] Doc fixes --- Doc/whatsnew/3.14.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index dca311880f9826..f91a9066871f06 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -517,6 +517,7 @@ Added a new command-line interface to inspect running Python processes using asynchronous tasks, available via: .. code-block:: bash + python -m asyncio.tools [--tree] PID This tool inspects the given process ID (PID) and displays information about From 0f75edcc0e6a3a5e6e7687c1dbee889613560736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 19:47:39 +0200 Subject: [PATCH 22/31] Fix doc build --- Doc/whatsnew/3.14.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index f91a9066871f06..0e19461d6e20a4 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -508,7 +508,7 @@ configuration mechanisms). .. seealso:: :pep:`741`. -.. _whatsnew314-asyncio-introspection +.. _whatsnew314-asyncio-introspection: Asyncio introspection capabilities ---------------------------------- @@ -597,7 +597,7 @@ If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that prevent tree construction. -(Contributed by Pablo Galindo, Łukasz Langa and Marta Gomez Macias in :gh:`91048`.) +(Contributed by Pablo Galindo, Łukasz Langa, and Marta Gomez Macias in :gh:`91048`.) .. _whatsnew314-tail-call: From c3a6bcbbda1a631496695e966bc3922480aa2135 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 20:01:11 +0200 Subject: [PATCH 23/31] Add Yury --- Doc/whatsnew/3.14.rst | 3 ++- .../2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 0e19461d6e20a4..5bb482de3685f5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -597,7 +597,8 @@ If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that prevent tree construction. -(Contributed by Pablo Galindo, Łukasz Langa, and Marta Gomez Macias in :gh:`91048`.) +(Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta +Gomez Macias in :gh:`91048`.) .. _whatsnew314-tail-call: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst index c33810614d4d56..0d5e7734f212d6 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst @@ -1,4 +1,5 @@ Add a new ``python -m asyncio.tools`` command-line interface to inspect asyncio tasks in a running Python process. Displays a flat table of await relationships or a tree view with ``--tree``, useful for debugging async -code. Patch by Pablo Galindo, Łukasz Langa and Marta Gomez Macias. +code. Patch by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta +Gomez Macias. From 5e1cb87c1d9cd3a1a4dadfedb570ee353f468011 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Sat, 3 May 2025 18:04:37 +0000 Subject: [PATCH 24/31] fix: Show independent tasks in the table --- Lib/asyncio/tools.py | 11 +++++ Lib/test/test_asyncio/test_tools.py | 72 ++++++++++++++++++++++++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 1719821a1ffe6e..4daace72471465 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -144,6 +144,17 @@ def build_task_table(result): table = [] for tid, tasks in result: for task_id, task_name, awaited in tasks: + if not awaited: + table.append( + [ + tid, + hex(task_id), + task_name, + "", + "", + "0x0" + ] + ) for stack, awaiter_id in awaited: coroutine_chain = " -> ".join(stack) awaiter_name = id2name.get(awaiter_id, "Unknown") diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 2f2b5feac43d74..c9abdc60387310 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -164,6 +164,37 @@ ] ), ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + (1, [(2, "Task-5", [])]), + ( + 3, + [ + (4, "Task-1", []), + (5, "Task-2", [[["main"], 4]]), + (6, "Task-3", [[["main"], 4]]), + (7, "Task-4", [[["main"], 4]]), + ], + ), + (8, []), + (0, []), + ] + ), + ( + [ + ["└── (T) Task-5"], + [ + "└── (T) Task-1", + " └── main", + " ├── (T) Task-2", + " ├── (T) Task-3", + " └── (T) Task-4", + ], + ] + ), + ], ] TEST_INPUTS_CYCLES_TREE = [ @@ -300,6 +331,7 @@ ), ( [ + [1, "0x2", "Task-1", "", "", "0x0"], [ 1, "0x3", @@ -409,15 +441,45 @@ ), ( [ + [9, "0x5", "Task-5", "", "", "0x0"], [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], [9, "0x7", "Task-7", "main2", "Task-5", "0x5"], [9, "0x8", "Task-8", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "0x0"], [10, "0x2", "Task-2", "main", "Task-1", "0x1"], [10, "0x3", "Task-3", "main", "Task-1", "0x1"], [10, "0x4", "Task-4", "main", "Task-1", "0x1"], ] ), ], + [ + # test case containing two roots, one of them without subtasks + ( + [ + (1, [(2, "Task-5", [])]), + ( + 3, + [ + (4, "Task-1", []), + (5, "Task-2", [[["main"], 4]]), + (6, "Task-3", [[["main"], 4]]), + (7, "Task-4", [[["main"], 4]]), + ], + ), + (8, []), + (0, []), + ] + ), + ( + [ + [1, "0x2", "Task-5", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "0x0"], + [3, "0x5", "Task-2", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "main", "Task-1", "0x4"], + ] + ), + ], # CASES WITH CYCLES [ # this test case contains a cycle: two tasks awaiting each other. @@ -440,6 +502,7 @@ ), ( [ + [1, "0x2", "Task-1", "", "", "0x0"], [1, "0x3", "a", "awaiter2", "b", "0x4"], [1, "0x3", "a", "main", "Task-1", "0x2"], [1, "0x4", "b", "awaiter", "a", "0x3"], @@ -480,6 +543,7 @@ ), ( [ + [1, "0x2", "Task-1", "", "", "0x0"], [ 1, "0x3", @@ -570,7 +634,10 @@ def test_only_independent_tasks_tree(self): def test_only_independent_tasks_table(self): input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] - self.assertEqual(tools.build_task_table(input_), []) + self.assertEqual( + tools.build_task_table(input_), + [[1, "0xa", "taskA", "", "", "0x0"], [1, "0xb", "taskB", "", "", "0x0"]], + ) def test_single_task_tree(self): """Test print_async_tree with a single task and no awaits.""" @@ -599,7 +666,7 @@ def test_single_task_table(self): ], ) ] - expected_output = [] + expected_output = [[1, "0x2", "Task-1", "", "", "0x0"]] self.assertEqual(tools.build_task_table(result), expected_output) def test_cycle_detection(self): @@ -653,6 +720,7 @@ def test_complex_table(self): ) ] expected_output = [ + [1, "0x2", "Task-1", "", "", "0x0"], [1, "0x3", "Task-2", "main", "Task-1", "0x2"], [1, "0x4", "Task-3", "main", "Task-2", "0x3"], ] From af6a8bf0210fef0ad18b593e320fa521a4080e36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 22:46:48 +0200 Subject: [PATCH 25/31] Temporarily skip test_async_global_awaited_by on free-threading --- Lib/test/test_external_inspection.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 0c8fc0395b65cc..4e82f567e1f429 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -5,7 +5,7 @@ import sys import socket from unittest.mock import ANY -from test.support import os_helper, SHORT_TIMEOUT, busy_retry +from test.support import os_helper, SHORT_TIMEOUT, busy_retry, requires_gil_enabled from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -406,6 +406,7 @@ async def main(): self.assertEqual(stack_trace, expected_stack_trace) @skip_if_not_supported + @requires_gil_enabled("gh-133359: occasionally flaky on AMD64") @unittest.skipIf(sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, "Test only runs on Linux with process_vm_readv support") def test_async_global_awaited_by(self): @@ -517,7 +518,6 @@ async def main(): self.assertEqual(all_awaited_by[1], (0, [])) entries = all_awaited_by[0][1] # expected: at least 1000 pending tasks - print(entries, file=sys.stderr) self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure self.assertIn((ANY, 'Task-1', []), entries) @@ -548,7 +548,6 @@ async def main(): "Test only runs on Linux with process_vm_readv support") def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) - print(stack_trace) self.assertEqual(stack_trace[0], "test_self_trace") if __name__ == "__main__": From 8db5dbe3f11030f8f8811fa0629eac710786bf15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 23:25:58 +0200 Subject: [PATCH 26/31] Drop the `tools`. It's cleaner. --- Doc/whatsnew/3.14.rst | 16 +++-- Lib/asyncio/__main__.py | 32 +++++++++ Lib/asyncio/tools.py | 68 +++++++++---------- Lib/test/test_asyncio/test_tools.py | 36 +++++----- ...5-05-03-19-04-03.gh-issue-91048.S8QWSw.rst | 5 +- 5 files changed, 96 insertions(+), 61 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5bb482de3685f5..65d0a319a07cf2 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -518,14 +518,18 @@ asynchronous tasks, available via: .. code-block:: bash - python -m asyncio.tools [--tree] PID + python -m asyncio ps PID This tool inspects the given process ID (PID) and displays information about -currently running asyncio tasks. By default, it outputs a task table: a flat +currently running asyncio tasks. It outputs a task table: a flat listing of all tasks, their names, their coroutine stacks, and which tasks are awaiting them. -With the ``--tree`` option, it instead renders a visual async call tree, +.. code-block:: bash + + python -m asyncio pstree PID + +This tool fetches the same information, but renders a visual async call tree, showing coroutine relationships in a hierarchical format. This command is particularly useful for debugging long-running or stuck asynchronous programs. It can help developers quickly identify where a program is blocked, what tasks @@ -565,7 +569,7 @@ Executing the new tool on the running process will yield a table like this: .. code-block:: bash - python -m asyncio.tools 12345 + python -m asyncio ps 12345 tid task id task name coroutine chain awaiter name awaiter id --------------------------------------------------------------------------------------------------------------------------------------- @@ -576,11 +580,11 @@ Executing the new tool on the running process will yield a table like this: 6826911 0x200013c0e20 Task-7 task_group Task-3 0x200013c0420 -and with the ``--tree`` option: +or: .. code-block:: bash - python -m asyncio.tools --tree 12345 + python -m asyncio pstree 12345 └── (T) Task-1 └── main diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 69f5a30cfe5095..7d980bc401ae3b 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -1,5 +1,7 @@ +import argparse import ast import asyncio +import asyncio.tools import concurrent.futures import contextvars import inspect @@ -140,6 +142,36 @@ def interrupt(self) -> None: if __name__ == '__main__': + parser = argparse.ArgumentParser( + prog="python3 -m asyncio", + description="Interactive asyncio shell and CLI tools", + ) + subparsers = parser.add_subparsers(help="sub-commands", dest="command") + ps = subparsers.add_parser( + "ps", help="Display a table of all pending tasks in a process" + ) + ps.add_argument("pid", type=int, help="Process ID to inspect") + pstree = subparsers.add_parser( + "pstree", help="Display a tree of all pending tasks in a process" + ) + pstree.add_argument("pid", type=int, help="Process ID to inspect") + args = parser.parse_args() + match args.command: + case "ps": + asyncio.tools.display_awaited_by_tasks_table(args.pid) + sys.exit(0) + case "pstree": + asyncio.tools.display_awaited_by_tasks_tree(args.pid) + sys.exit(0) + case None: + pass # continue to the interactive shell + case _: + # shouldn't happen as an invalid command-line wouldn't parse + # but let's keep it for the next person adding a command + print(f"error: unhandled command {args.command}", file=sys.stderr) + parser.print_usage(file=sys.stderr) + sys.exit(1) + sys.audit("cpython.run_stdin") if os.getenv('PYTHON_BASIC_REPL'): diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 4daace72471465..6d59ea4ebebf1c 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,4 +1,3 @@ -import argparse from dataclasses import dataclass from collections import defaultdict from itertools import count @@ -107,10 +106,12 @@ def dfs(v): # ─── PRINT TREE FUNCTION ─────────────────────────────────────── -def print_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): +def build_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ - Pretty-print the async call tree produced by `get_all_async_stacks()`, - prefixing tasks with *task_emoji* and coroutine frames with *cor_emoji*. + Build a list of strings for pretty-print a async call tree. + + The call tree is produced by `get_all_async_stacks()`, prefixing tasks + with `task_emoji` and coroutine frames with `cor_emoji`. """ id2name, awaits = _index(result) g = _task_graph(awaits) @@ -179,40 +180,39 @@ def _print_cycle_exception(exception: CycleFoundException): print(f"cycle: {inames}", file=sys.stderr) - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Show Python async tasks in a process") - parser.add_argument("pid", type=int, help="Process ID(s) to inspect.") - parser.add_argument( - "--tree", "-t", action="store_true", help="Display tasks in a tree format" - ) - args = parser.parse_args() - +def _get_awaited_by_tasks(pid: int) -> list: try: - tasks = get_all_awaited_by(args.pid) + return get_all_awaited_by(pid) except RuntimeError as e: while e.__context__ is not None: e = e.__context__ print(f"Error retrieving tasks: {e}") sys.exit(1) - if args.tree: - # Print the async call tree - try: - result = print_async_tree(tasks) - except CycleFoundException as e: - _print_cycle_exception(e) - sys.exit(1) - - for tree in result: - print("\n".join(tree)) - else: - # Build and print the task table - table = build_task_table(tasks) - # Print the table in a simple tabular format - print( - f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" - ) - print("-" * 135) - for row in table: - print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") + +def display_awaited_by_tasks_table(pid: int) -> None: + """Build and print a table of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + table = build_task_table(tasks) + # Print the table in a simple tabular format + print( + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" + ) + print("-" * 135) + for row in table: + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") + + +def display_awaited_by_tasks_tree(pid: int) -> None: + """Build and print a tree of all pending tasks under `pid`.""" + + tasks = _get_awaited_by_tasks(pid) + try: + result = print_async_tree(tasks) + except CycleFoundException as e: + _print_cycle_exception(e) + sys.exit(1) + + for tree in result: + print("\n".join(tree)) diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index c9abdc60387310..2caf56172c9193 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -1,5 +1,3 @@ -"""Tests for the asyncio tools script.""" - import unittest from asyncio import tools @@ -595,13 +593,13 @@ class TestAsyncioToolsTree(unittest.TestCase): def test_asyncio_utils(self): for input_, tree in TEST_INPUTS_TREE: with self.subTest(input_): - self.assertEqual(tools.print_async_tree(input_), tree) + self.assertEqual(tools.build_async_tree(input_), tree) def test_asyncio_utils_cycles(self): for input_, cycles in TEST_INPUTS_CYCLES_TREE: with self.subTest(input_): try: - tools.print_async_tree(input_) + tools.build_async_tree(input_) except tools.CycleFoundException as e: self.assertEqual(e.cycles, cycles) @@ -615,10 +613,10 @@ def test_asyncio_utils(self): class TestAsyncioToolsBasic(unittest.TestCase): def test_empty_input_tree(self): - """Test print_async_tree with empty input.""" + """Test build_async_tree with empty input.""" result = [] expected_output = [] - self.assertEqual(tools.print_async_tree(result), expected_output) + self.assertEqual(tools.build_async_tree(result), expected_output) def test_empty_input_table(self): """Test build_task_table with empty input.""" @@ -629,7 +627,7 @@ def test_empty_input_table(self): def test_only_independent_tasks_tree(self): input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] expected = [["└── (T) taskA"], ["└── (T) taskB"]] - result = tools.print_async_tree(input_) + result = tools.build_async_tree(input_) self.assertEqual(sorted(result), sorted(expected)) def test_only_independent_tasks_table(self): @@ -640,7 +638,7 @@ def test_only_independent_tasks_table(self): ) def test_single_task_tree(self): - """Test print_async_tree with a single task and no awaits.""" + """Test build_async_tree with a single task and no awaits.""" result = [ ( 1, @@ -654,7 +652,7 @@ def test_single_task_tree(self): "└── (T) Task-1", ] ] - self.assertEqual(tools.print_async_tree(result), expected_output) + self.assertEqual(tools.build_async_tree(result), expected_output) def test_single_task_table(self): """Test build_task_table with a single task and no awaits.""" @@ -670,7 +668,7 @@ def test_single_task_table(self): self.assertEqual(tools.build_task_table(result), expected_output) def test_cycle_detection(self): - """Test print_async_tree raises CycleFoundException for cyclic input.""" + """Test build_async_tree raises CycleFoundException for cyclic input.""" result = [ ( 1, @@ -681,11 +679,11 @@ def test_cycle_detection(self): ) ] with self.assertRaises(tools.CycleFoundException) as context: - tools.print_async_tree(result) + tools.build_async_tree(result) self.assertEqual(context.exception.cycles, [[3, 2, 3]]) def test_complex_tree(self): - """Test print_async_tree with a more complex tree structure.""" + """Test build_async_tree with a more complex tree structure.""" result = [ ( 1, @@ -705,7 +703,7 @@ def test_complex_tree(self): " └── (T) Task-3", ] ] - self.assertEqual(tools.print_async_tree(result), expected_output) + self.assertEqual(tools.build_async_tree(result), expected_output) def test_complex_table(self): """Test build_task_table with a more complex tree structure.""" @@ -747,7 +745,7 @@ def test_deep_coroutine_chain(self): " └── (T) leaf", ] ] - result = tools.print_async_tree(input_) + result = tools.build_async_tree(input_) self.assertEqual(result, expected) def test_multiple_cycles_same_node(self): @@ -762,7 +760,7 @@ def test_multiple_cycles_same_node(self): ) ] with self.assertRaises(tools.CycleFoundException) as ctx: - tools.print_async_tree(input_) + tools.build_async_tree(input_) cycles = ctx.exception.cycles self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) @@ -789,7 +787,7 @@ def test_task_awaits_self(self): """A task directly awaits itself – should raise a cycle.""" input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])] with self.assertRaises(tools.CycleFoundException) as ctx: - tools.print_async_tree(input_) + tools.build_async_tree(input_) self.assertIn([1, 1], ctx.exception.cycles) def test_task_with_missing_awaiter_id(self): @@ -811,7 +809,7 @@ def test_duplicate_coroutine_frames(self): ], ) ] - tree = tools.print_async_tree(input_) + tree = tools.build_async_tree(input_) # Both children should be under the same coroutine node flat = "\n".join(tree[0]) self.assertIn("frameA", flat) @@ -827,13 +825,13 @@ def test_task_with_no_name(self): """Task with no name in id2name – should still render with fallback.""" input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])] # If name is None, fallback to string should not crash - tree = tools.print_async_tree(input_) + tree = tools.build_async_tree(input_) self.assertIn("(T) None", "\n".join(tree[0])) def test_tree_rendering_with_custom_emojis(self): """Pass custom emojis to the tree renderer.""" input_ = [(1, [(1, "MainTask", [[["f1", "f2"], 2]]), (2, "SubTask", [])])] - tree = tools.print_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") + tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") flat = "\n".join(tree[0]) self.assertIn("🧵 MainTask", flat) self.assertIn("🔁 f1", flat) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst index 0d5e7734f212d6..1d45868b7b27bc 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-03-19-04-03.gh-issue-91048.S8QWSw.rst @@ -1,5 +1,6 @@ -Add a new ``python -m asyncio.tools`` command-line interface to inspect +Add a new ``python -m asyncio ps PID`` command-line interface to inspect asyncio tasks in a running Python process. Displays a flat table of await -relationships or a tree view with ``--tree``, useful for debugging async +relationships. A variant showing a tree view is also available as +``python -m asyncio pstree PID``. Both are useful for debugging async code. Patch by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta Gomez Macias. From 6f8aa6b441d95b9ffbeb9f360e007c469433b488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Sat, 3 May 2025 23:31:41 +0200 Subject: [PATCH 27/31] Satisfy the linting gods --- Lib/asyncio/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 6d59ea4ebebf1c..8b63b80061203a 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -109,7 +109,7 @@ def dfs(v): def build_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): """ Build a list of strings for pretty-print a async call tree. - + The call tree is produced by `get_all_async_stacks()`, prefixing tasks with `task_emoji` and coroutine frames with `cor_emoji`. """ From 8d566c6bc915578e98f44196fa3b4d1753b225a1 Mon Sep 17 00:00:00 2001 From: Marta Gomez Macias Date: Sun, 4 May 2025 00:00:22 +0000 Subject: [PATCH 28/31] chore: Refactor --- Lib/asyncio/tools.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 8b63b80061203a..ee96dd80b8f04f 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,3 +1,5 @@ +"""Tool to analyze tasks running in a asyncio script.""" + from dataclasses import dataclass from collections import defaultdict from itertools import count @@ -46,10 +48,6 @@ def _cor_node(parent_key, frame_name): bucket[frame_name] = node_key return node_key - # touch every task so it’s present even if it awaits nobody - for tid in id2name: - children[(NodeType.TASK, tid)] - # lay down parent ➜ …frames… ➜ child paths for parent_id, stack, child_id in awaits: cur = (NodeType.TASK, parent_id) @@ -69,7 +67,6 @@ def _roots(id2label, children): # ─── detect cycles in the task-to-task graph ─────────────────────── def _task_graph(awaits): """Return {parent_task_id: {child_task_id, …}, …}.""" - from collections import defaultdict g = defaultdict(set) for parent_id, _stack, child_id in awaits: g[parent_id].add(child_id) @@ -106,7 +103,7 @@ def dfs(v): # ─── PRINT TREE FUNCTION ─────────────────────────────────────── -def build_async_tree(result, task_emoji="(T)", cor_emoji="", printer=print): +def build_async_tree(result, task_emoji="(T)", cor_emoji=""): """ Build a list of strings for pretty-print a async call tree. @@ -134,10 +131,7 @@ def render(node, prefix="", last=True, buf=None): render(kid, new_pref, i == len(kids) - 1, buf) return buf - result = [] - for r, root in enumerate(_roots(labels, children)): - result.append(render(root)) - return result + return [render(root) for root in _roots(labels, children)] def build_task_table(result): @@ -209,7 +203,7 @@ def display_awaited_by_tasks_tree(pid: int) -> None: tasks = _get_awaited_by_tasks(pid) try: - result = print_async_tree(tasks) + result = build_async_tree(tasks) except CycleFoundException as e: _print_cycle_exception(e) sys.exit(1) From 9dbe00d276b3bd8eb1201e1450a2db63239739bf Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 4 May 2025 02:17:02 +0200 Subject: [PATCH 29/31] Doc fixes --- Doc/whatsnew/3.14.rst | 67 ++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 65d0a319a07cf2..33bfc783480973 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -541,26 +541,21 @@ For example given this code: import asyncio - async def sleeper(name, delay): - await asyncio.sleep(delay) - print(f"{name} is done sleeping.") + async def play(track): + await asyncio.sleep(5) + print(f"🎵 Finished: {track}") - async def inner(name): - await asyncio.sleep(0.1) - await sleeper(name, 1) - - async def task_group(name): - await asyncio.gather( - inner(f"{name}-1"), - inner(f"{name}-2"), - ) + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) async def main(): - # Start two separate task groups - t1 = asyncio.create_task(task_group("groupA")) - t2 = asyncio.create_task(task_group("groupB")) - await t1 - await t2 + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), name="Sundowning") + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), name="TMBTE") if __name__ == "__main__": asyncio.run(main()) @@ -571,13 +566,15 @@ Executing the new tool on the running process will yield a table like this: python -m asyncio ps 12345 - tid task id task name coroutine chain awaiter name awaiter id + tid task id task name coroutine chain awaiter name awaiter id --------------------------------------------------------------------------------------------------------------------------------------- - 6826911 0x200013c0220 Task-2 main Task-1 0x200013b0020 - 6826911 0x200013c0620 Task-4 task_group Task-2 0x200013c0220 - 6826911 0x200013c0820 Task-5 task_group Task-2 0x200013c0220 - 6826911 0x200013c0c20 Task-6 task_group Task-3 0x200013c0420 - 6826911 0x200013c0e20 Task-7 task_group Task-3 0x200013c0420 + 8138752 0x564bd3d0210 Task-1 0x0 + 8138752 0x564bd3d0410 Sundowning _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 + 8138752 0x564bd3d0610 TMBTE _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 + 8138752 0x564bd3d0810 TNDNBTG _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 + 8138752 0x564bd3d0a10 Levitate _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 + 8138752 0x564bd3e0550 DYWTYLM _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 + 8138752 0x564bd3e0710 Aqua Regia _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 or: @@ -587,15 +584,21 @@ or: python -m asyncio pstree 12345 └── (T) Task-1 - └── main - └── (T) Task-2 - └── task_group - ├── (T) Task-4 - └── (T) Task-5 - └── (T) Task-3 - └── task_group - ├── (T) Task-6 - └── (T) Task-7 + └── main + └── __aexit__ + └── _aexit + ├── (T) Sundowning + │ └── album + │ └── __aexit__ + │ └── _aexit + │ ├── (T) TNDNBTG + │ └── (T) Levitate + └── (T) TMBTE + └── album + └── __aexit__ + └── _aexit + ├── (T) DYWTYLM + └── (T) Aqua Regia If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that From c56782beac2a0d255be2b4933091d56e39311eab Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 4 May 2025 02:21:45 +0200 Subject: [PATCH 30/31] Type fixes --- Modules/_remotedebuggingmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_remotedebuggingmodule.c b/Modules/_remotedebuggingmodule.c index f6e8135a6c5800..0e055ae1604d5f 100644 --- a/Modules/_remotedebuggingmodule.c +++ b/Modules/_remotedebuggingmodule.c @@ -517,7 +517,7 @@ parse_task( tn = parse_task_name( handle, offsets, async_offsets, task_address); } else { - tn = PyLong_FromUnsignedLong(task_address); + tn = PyLong_FromUnsignedLongLong(task_address); } if (tn == NULL) { goto err; @@ -1079,7 +1079,7 @@ append_awaited_by_for_thread( return -1; } - PyObject* task_id = PyLong_FromUnsignedLong(task_addr); + PyObject* task_id = PyLong_FromUnsignedLongLong(task_addr); if (task_id == NULL) { Py_DECREF(tn); Py_DECREF(current_awaited_by); From 293337f560b4df15ea4f7d67c4c937cf66717ca1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sun, 4 May 2025 02:24:33 +0200 Subject: [PATCH 31/31] Type fixes --- Lib/asyncio/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index ee96dd80b8f04f..16440b594ad993 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,4 +1,4 @@ -"""Tool to analyze tasks running in a asyncio script.""" +"""Tools to analyze tasks running in asyncio programs.""" from dataclasses import dataclass from collections import defaultdict pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy