From 8040f9aae244986ea10a3a7f4de6719400a58efd Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Sat, 24 May 2025 15:11:45 +1200 Subject: [PATCH 1/7] gh-109934: notify cancelled futures on thread pool shutdown When `ThreadPoolExecutor` shuts down it cancels any pending futures, however at present it doesn't notify waiters. Thus their state stays as `CANCELLED` instead of `CANCELLED_AND_NOTIFIED` and any waiters are not awakened. Call `set_running_or_notify_cancel` on the cancelled futures to fix this. --- Lib/concurrent/futures/thread.py | 1 + .../test_thread_pool.py | 50 +++++++++++++++++++ ...-05-24-15-15-43.gh-issue-109934.WXOdC8.rst | 2 + 3 files changed, 53 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst diff --git a/Lib/concurrent/futures/thread.py b/Lib/concurrent/futures/thread.py index 909359b648709f..bd8eb33d753598 100644 --- a/Lib/concurrent/futures/thread.py +++ b/Lib/concurrent/futures/thread.py @@ -264,6 +264,7 @@ def shutdown(self, wait=True, *, cancel_futures=False): break if work_item is not None: work_item.future.cancel() + work_item.future.set_running_or_notify_cancel() # Send a wake-up to prevent threads calling # _work_queue.get(block=True) from permanently blocking. diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 4324241b374967..ad3488f0407432 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -112,6 +112,56 @@ def log_n_wait(ident): # ident='third' is cancelled because it remained in the collection of futures self.assertListEqual(log, ["ident='first' started", "ident='first' stopped"]) + def test_shutdown_cancels_pending_futures(self): + def waiter(barrier): + barrier.wait(3) + def noop(): + pass + barrier = threading.Barrier(2) + called_back_1 = threading.Event() + called_back_2 = threading.Event() + with self.executor_type(max_workers=1) as pool: + + # Submit two futures, the first of which will block and prevent the + # second from running + f1 = pool.submit(waiter, barrier) + f2 = pool.submit(noop) + f1.add_done_callback(lambda f: called_back_1.set()) + f2.add_done_callback(lambda f: called_back_2.set()) + fs = {f1, f2} + + completed_iter = futures.as_completed(fs, timeout=0) + self.assertRaises(TimeoutError, next, completed_iter) + + # Shutdown the pool, cancelling unstarted task + pool.shutdown(wait=False, cancel_futures=True) + self.assertTrue(f1.running()) + self.assertTrue(f2.cancelled()) + self.assertFalse(called_back_1.is_set()) + self.assertTrue(called_back_2.is_set()) + + completed_iter = futures.as_completed(fs, timeout=0) + f = next(completed_iter) + self.assertIs(f, f2) + self.assertRaises(TimeoutError, next, completed_iter) + + result = futures.wait(fs, timeout=0) + self.assertEqual(result.not_done, {f1}) + self.assertEqual(result.done, {f2}) + + # Unblock and wait for the first future to complete + barrier.wait(3) + called_back_1.wait(3) + self.assertTrue(f1.done()) + self.assertTrue(called_back_1.is_set()) + + completed = set(futures.as_completed(fs, timeout=0)) + self.assertEqual(set(fs), completed) + + result = futures.wait(fs, timeout=0) + self.assertEqual(result.not_done, set()) + self.assertEqual(result.done, set(fs)) + def setUpModule(): setup_module() diff --git a/Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst b/Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst new file mode 100644 index 00000000000000..85de1993909453 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst @@ -0,0 +1,2 @@ +Ensure :class:`concurrent.futures.ThreadPoolExecutor` notifies any futures +it cancels on shutdown. From bb89020fef4f77663add656b9b8b9df3a8df45ac Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Tue, 15 Jul 2025 22:35:48 +1200 Subject: [PATCH 2/7] Add reference to GH issue --- Lib/test/test_concurrent_futures/test_thread_pool.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index ad3488f0407432..876732b5cdf349 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -113,6 +113,7 @@ def log_n_wait(ident): self.assertListEqual(log, ["ident='first' started", "ident='first' stopped"]) def test_shutdown_cancels_pending_futures(self): + # gh-109934: ensure shutdown cancels and notifies pending futures def waiter(barrier): barrier.wait(3) def noop(): From ff7084307ee248fcd85f92dd31f74b5e71aee74e Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Tue, 15 Jul 2025 22:35:56 +1200 Subject: [PATCH 3/7] gh-136655: ensure cancelled futures are notified on process pool shutdown --- Lib/concurrent/futures/process.py | 4 +++- Lib/test/test_concurrent_futures/executor.py | 22 ++++++++++++++++++++ Lib/test/test_concurrent_futures/util.py | 15 +++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 76b7b2abe836d8..73c38776094057 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -518,7 +518,9 @@ def flag_executor_shutting_down(self): # to only have futures that are currently running. new_pending_work_items = {} for work_id, work_item in self.pending_work_items.items(): - if not work_item.future.cancel(): + if work_item.future.cancel(): + work_item.future.set_running_or_notify_cancel() + else: new_pending_work_items[work_id] = work_item self.pending_work_items = new_pending_work_items # Drain work_ids_queue since we no longer need to diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 95bf8fcd25bf54..6a3c03e32b0c31 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -233,3 +233,25 @@ def test_swallows_falsey_exceptions(self): msg = 'lenlen' with self.assertRaisesRegex(FalseyLenException, msg): self.executor.submit(raiser, FalseyLenException, msg).result() + + def test_shutdown_notifies_cancelled_futures(self): + # gh-136655: ensure cancelled futures are notified + count = self.worker_count * 2 + barrier = self.create_barrier(self.worker_count + 1) + fs = [self.executor.submit(blocking_raiser, + barrier if index < self.worker_count else None) + for index in range(count)] + + self.executor.shutdown(wait=False, cancel_futures=True) + barrier.wait() + + for future in fs: + self.assertRaises((FalseyBoolException, futures.CancelledError), + future.result) + + self.assertIn('CANCELLED_AND_NOTIFIED', [f._state for f in fs]) + +def blocking_raiser(barrier=None): + if barrier is not None: + barrier.wait(1) + raise FalseyBoolException() diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index b12940414d9142..aceb51cacfae26 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -79,6 +79,9 @@ def get_context(self): class ThreadPoolMixin(ExecutorMixin): executor_type = futures.ThreadPoolExecutor + def create_barrier(self, count): + return threading.Barrier(count) + def create_event(self): return threading.Event() @@ -87,6 +90,9 @@ def create_event(self): class InterpreterPoolMixin(ExecutorMixin): executor_type = futures.InterpreterPoolExecutor + def create_barrier(self, count): + self.skipTest("InterpreterPoolExecutor doesn't support barriers") + def create_event(self): self.skipTest("InterpreterPoolExecutor doesn't support events") @@ -106,6 +112,9 @@ def get_context(self): self.skipTest("TSAN doesn't support threads after fork") return super().get_context() + def create_barrier(self, count): + return self.manager.Barrier(count) + def create_event(self): return self.manager.Event() @@ -121,6 +130,9 @@ def get_context(self): self.skipTest("ProcessPoolExecutor unavailable on this system") return super().get_context() + def create_barrier(self, count): + return self.manager.Barrier(count) + def create_event(self): return self.manager.Event() @@ -140,6 +152,9 @@ def get_context(self): self.skipTest("TSAN doesn't support threads after fork") return super().get_context() + def create_barrier(self, count): + return self.manager.Barrier(count) + def create_event(self): return self.manager.Event() From df11bf398e2b33bdadb3da783b084dbb2a90db4c Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Tue, 15 Jul 2025 23:15:17 +1200 Subject: [PATCH 4/7] Ignore broken barrier errors when waiting on the barrier in the main process --- Lib/test/test_concurrent_futures/executor.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 6a3c03e32b0c31..c37ab5b09ad96c 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -243,7 +243,10 @@ def test_shutdown_notifies_cancelled_futures(self): for index in range(count)] self.executor.shutdown(wait=False, cancel_futures=True) - barrier.wait() + try: + barrier.wait() + except threading.BrokenBarrierError: + pass for future in fs: self.assertRaises((FalseyBoolException, futures.CancelledError), From ffdcb27748200939505d49d48a850d792d29e06a Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Wed, 16 Jul 2025 10:40:36 +1200 Subject: [PATCH 5/7] Catch BrokenBarrierError in another place it can be thrown and mention the ProcessPoolExecutor in NEWS. --- Lib/test/test_concurrent_futures/executor.py | 5 +++-- .../Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index c37ab5b09ad96c..de7f6cb2b80dd9 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -249,8 +249,9 @@ def test_shutdown_notifies_cancelled_futures(self): pass for future in fs: - self.assertRaises((FalseyBoolException, futures.CancelledError), - future.result) + self.assertRaises( + (FalseyBoolException, futures.CancelledError, threading.BrokenBarrierError), + future.result) self.assertIn('CANCELLED_AND_NOTIFIED', [f._state for f in fs]) diff --git a/Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst b/Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst index 85de1993909453..21b097f9b72e13 100644 --- a/Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst +++ b/Misc/NEWS.d/next/Library/2025-05-24-15-15-43.gh-issue-109934.WXOdC8.rst @@ -1,2 +1,3 @@ -Ensure :class:`concurrent.futures.ThreadPoolExecutor` notifies any futures -it cancels on shutdown. +Ensure :class:`concurrent.futures.ThreadPoolExecutor` and +:class:`concurrent.futures.ProcessPoolExecutor` notifies any futures it cancels +on shutdown. From 8e2b57364c4b883a48355a21a83e25f2736d7351 Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Wed, 16 Jul 2025 11:52:15 +1200 Subject: [PATCH 6/7] Fix another race condition in the threading executor test: ensure the blocking future has started before checking its status. --- .../test_concurrent_futures/test_thread_pool.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_concurrent_futures/test_thread_pool.py b/Lib/test/test_concurrent_futures/test_thread_pool.py index 876732b5cdf349..ab270a91670f10 100644 --- a/Lib/test/test_concurrent_futures/test_thread_pool.py +++ b/Lib/test/test_concurrent_futures/test_thread_pool.py @@ -114,18 +114,20 @@ def log_n_wait(ident): def test_shutdown_cancels_pending_futures(self): # gh-109934: ensure shutdown cancels and notifies pending futures - def waiter(barrier): - barrier.wait(3) + def waiter(b1, b2): + b1.wait(3) + b2.wait(3) def noop(): pass - barrier = threading.Barrier(2) + b1 = threading.Barrier(2) + b2 = threading.Barrier(2) called_back_1 = threading.Event() called_back_2 = threading.Event() with self.executor_type(max_workers=1) as pool: # Submit two futures, the first of which will block and prevent the # second from running - f1 = pool.submit(waiter, barrier) + f1 = pool.submit(waiter, b1, b2) f2 = pool.submit(noop) f1.add_done_callback(lambda f: called_back_1.set()) f2.add_done_callback(lambda f: called_back_2.set()) @@ -134,7 +136,9 @@ def noop(): completed_iter = futures.as_completed(fs, timeout=0) self.assertRaises(TimeoutError, next, completed_iter) - # Shutdown the pool, cancelling unstarted task + # Ensure the first task has started running then shutdown the + # pool, cancelling the unstarted task + b1.wait(3) pool.shutdown(wait=False, cancel_futures=True) self.assertTrue(f1.running()) self.assertTrue(f2.cancelled()) @@ -151,7 +155,7 @@ def noop(): self.assertEqual(result.done, {f2}) # Unblock and wait for the first future to complete - barrier.wait(3) + b2.wait(3) called_back_1.wait(3) self.assertTrue(f1.done()) self.assertTrue(called_back_1.is_set()) From bc3751e31402f1520056d80fc86b23e91178bd01 Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Wed, 16 Jul 2025 22:40:16 +1200 Subject: [PATCH 7/7] Attempt to make the unit test more robust and clean up process management resources. --- Lib/test/test_concurrent_futures/executor.py | 33 ++++++++++---------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index de7f6cb2b80dd9..9a993fe32aa69a 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -238,22 +238,23 @@ def test_shutdown_notifies_cancelled_futures(self): # gh-136655: ensure cancelled futures are notified count = self.worker_count * 2 barrier = self.create_barrier(self.worker_count + 1) - fs = [self.executor.submit(blocking_raiser, - barrier if index < self.worker_count else None) - for index in range(count)] - - self.executor.shutdown(wait=False, cancel_futures=True) - try: - barrier.wait() - except threading.BrokenBarrierError: - pass - - for future in fs: - self.assertRaises( - (FalseyBoolException, futures.CancelledError, threading.BrokenBarrierError), - future.result) - - self.assertIn('CANCELLED_AND_NOTIFIED', [f._state for f in fs]) + with self.executor as exec: + fs = [exec.submit(blocking_raiser, + barrier if index < self.worker_count else None) + for index in range(count)] + + exec.shutdown(wait=False, cancel_futures=True) + try: + barrier.wait() + except threading.BrokenBarrierError: + pass + + for future in fs: + self.assertRaises( + (FalseyBoolException, futures.CancelledError, threading.BrokenBarrierError), + future.result) + + self.assertIn('CANCELLED_AND_NOTIFIED', [f._state for f in fs]) def blocking_raiser(barrier=None): if barrier is not None: pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy