diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a1259ff1d63d18..87194cc512a253 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -39,7 +39,7 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper - +from test.support.script_helper import assert_python_failure, assert_python_ok # Skip tests if _multiprocessing wasn't built. _multiprocessing = import_helper.import_module('_multiprocessing') @@ -325,6 +325,7 @@ def test_current(self): self.assertEqual(current.ident, os.getpid()) self.assertEqual(current.exitcode, None) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_set_executable(self): if self.TYPE == 'threads': self.skipTest(f'test not appropriate for {self.TYPE}') @@ -341,6 +342,7 @@ def test_set_executable(self): p.join() self.assertEqual(p.exitcode, 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_resource('cpu') def test_args_argument(self): # bpo-45735: Using list or tuple as *args* in constructor could @@ -388,6 +390,7 @@ def _test(cls, q, *args, **kwds): q.put(bytes(current.authkey)) q.put(current.pid) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_parent_process_attributes(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -408,6 +411,7 @@ def _test_send_parent_process(cls, wconn): from multiprocessing.process import parent_process wconn.send([parent_process().pid, parent_process().name]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_parent_process(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -446,6 +450,7 @@ def _test_report_parent_status(cls, wconn): parent_process().join(timeout=support.SHORT_TIMEOUT) wconn.send("alive" if parent_process().is_alive() else "not alive") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_process(self): q = self.Queue(1) e = self.Event() @@ -486,6 +491,7 @@ def test_process(self): self.assertNotIn(p, self.active_children()) close_queue(q) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(threading._HAVE_THREAD_NATIVE_ID, "needs native_id") def test_process_mainthread_native_id(self): if self.TYPE == 'threads': @@ -526,6 +532,7 @@ def _sleep_no_int_handler(cls, event): def _test_sleep(cls, delay): time.sleep(delay) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def _kill_process(self, meth, target=None): if self.TYPE == 'threads': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -583,6 +590,7 @@ def handler(*args): return p.exitcode + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipIf(os.name == 'nt', "POSIX only") def test_interrupt(self): exitcode = self._kill_process(multiprocessing.Process.interrupt) @@ -591,15 +599,18 @@ def test_interrupt(self): # (KeyboardInterrupt in this case) # in multiprocessing.BaseProcess._bootstrap + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipIf(os.name == 'nt', "POSIX only") def test_interrupt_no_handler(self): exitcode = self._kill_process(multiprocessing.Process.interrupt, target=self._sleep_no_int_handler) self.assertEqual(exitcode, -signal.SIGINT) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_terminate(self): exitcode = self._kill_process(multiprocessing.Process.terminate) self.assertEqual(exitcode, -signal.SIGTERM) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_kill(self): exitcode = self._kill_process(multiprocessing.Process.kill) if os.name != 'nt': @@ -615,6 +626,7 @@ def test_cpu_count(self): self.assertIsInstance(cpus, int) self.assertGreaterEqual(cpus, 1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_active_children(self): self.assertEqual(type(self.active_children()), list) @@ -643,6 +655,7 @@ def _test_recursion(cls, wconn, id): p.start() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_recursion(self): rconn, wconn = self.Pipe(duplex=False) self._test_recursion(wconn, []) @@ -667,6 +680,7 @@ def test_recursion(self): def _test_sentinel(cls, event): event.wait(10.0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_sentinel(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -689,6 +703,7 @@ def _test_close(cls, rc=0, q=None): q.get() sys.exit(rc) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_close(self): if self.TYPE == "threads": self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -721,6 +736,7 @@ def test_close(self): close_queue(q) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_resource('walltime') def test_many_processes(self): if self.TYPE == 'threads': @@ -757,6 +773,7 @@ def test_many_processes(self): for p in procs: self.assertIn(p.exitcode, exitcodes) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_lose_target_ref(self): c = DummyCallable() wr = weakref.ref(c) @@ -819,6 +836,7 @@ def func2(): threading.Thread(target=func1).start() threading.Thread(target=func2, daemon=True).start() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_wait_for_threads(self): # A child process should wait for non-daemonic threads to end # before exiting @@ -843,6 +861,7 @@ def _test_error_on_stdio_flush(self, evt, break_std_streams={}): setattr(sys, stream_name, None) evt.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_error_on_stdio_flush_1(self): # Check that Process works with broken standard streams streams = [io.StringIO(), None] @@ -862,6 +881,7 @@ def test_error_on_stdio_flush_1(self): finally: setattr(sys, stream_name, old_stream) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_error_on_stdio_flush_2(self): # Same as test_error_on_stdio_flush_1(), but standard streams are # broken by the child process @@ -1012,6 +1032,7 @@ class _TestSubclassingProcess(BaseTestCase): ALLOWED_TYPES = ('processes',) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_subclassing(self): uppercaser = _UpperCaser() uppercaser.daemon = True @@ -1021,6 +1042,7 @@ def test_subclassing(self): uppercaser.stop() uppercaser.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_stderr_flush(self): # sys.stderr is flushed at process shutdown (issue #13812) if self.TYPE == "threads": @@ -1051,6 +1073,7 @@ def _test_sys_exit(cls, reason, testfn): sys.stderr = open(fd, 'w', encoding="utf-8", closefd=False) sys.exit(reason) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_sys_exit(self): # See Issue 13854 if self.TYPE == 'threads': @@ -1118,6 +1141,7 @@ def _test_put(cls, queue, child_can_start, parent_can_continue): queue.get() parent_can_continue.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_put(self): MAXSIZE = 6 queue = self.Queue(maxsize=MAXSIZE) @@ -1187,6 +1211,7 @@ def _test_get(cls, queue, child_can_start, parent_can_continue): queue.put(5) parent_can_continue.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_get(self): queue = self.Queue() child_can_start = self.Event() @@ -1248,6 +1273,7 @@ def _test_fork(cls, queue): # process cannot shutdown until the feeder thread has finished # pushing items onto the pipe. + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_fork(self): # Old versions of Queue would fail to create a new feeder # thread for a forked process if the original process had its @@ -1298,6 +1324,7 @@ def _test_task_done(cls, q): time.sleep(DELTA) q.task_done() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_task_done(self): queue = self.JoinableQueue() @@ -1341,6 +1368,7 @@ def test_no_import_lock_contention(self): self.fail("Probable regression on import lock contention;" " see Issue #22853") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_timeout(self): q = multiprocessing.Queue() start = time.monotonic() @@ -1464,6 +1492,7 @@ def _acquire_event(lock, event): event.set() time.sleep(1.0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_repr_lock(self): if self.TYPE != 'processes': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -1527,6 +1556,7 @@ def _test_lock_locked_2processes(cls, lock, event, res): res.value = lock.locked() event.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_lock_locked_2processes(self): if self.TYPE != 'processes': @@ -1553,6 +1583,7 @@ def _acquire_release(lock, timeout, l=None, n=1): for _ in range(n): lock.release() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_repr_rlock(self): if self.TYPE != 'processes': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -1612,6 +1643,7 @@ def test_rlock(self): self.assertFalse(lock.locked()) self.assertRaises((AssertionError, RuntimeError), lock.release) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_rlock_locked_2processes(self): if self.TYPE != 'processes': @@ -1723,6 +1755,7 @@ def check_invariant(self, cond): except NotImplementedError: pass + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_notify(self): cond = self.Condition() sleeping = self.Semaphore(0) @@ -1765,6 +1798,7 @@ def test_notify(self): threading_helper.join_thread(t) join_process(p) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_notify_all(self): cond = self.Condition() sleeping = self.Semaphore(0) @@ -1834,6 +1868,7 @@ def test_notify_all(self): # NOTE: join_process and join_thread are the same threading_helper.join_thread(w) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_notify_n(self): cond = self.Condition() sleeping = self.Semaphore(0) @@ -1907,6 +1942,7 @@ def _test_waitfor_f(cls, cond, state): if not result or state.value != 4: sys.exit(1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_waitfor(self): # based on test in test/lock_tests.py @@ -1942,6 +1978,7 @@ def _test_waitfor_timeout_f(cls, cond, state, success, sem): if not result and (expected - CLOCK_RES) <= dt: success.value = True + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes') def test_waitfor_timeout(self): # based on test in test/lock_tests.py @@ -1974,6 +2011,7 @@ def _test_wait_result(cls, c, pid): if pid is not None: os.kill(pid, signal.SIGINT) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_wait_result(self): if isinstance(self, ProcessesMixin) and sys.platform != 'win32': pid = os.getpid() @@ -2002,6 +2040,7 @@ def _test_event(cls, event): time.sleep(TIMEOUT2) event.set() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_event(self): event = self.Event() wait = TimingWrapper(event.wait) @@ -2198,6 +2237,7 @@ def multipass(cls, barrier, results, n): pass assert not barrier.broken + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_barrier(self, passes=1): """ Test that a barrier is passed in lockstep @@ -2205,6 +2245,7 @@ def test_barrier(self, passes=1): results = [self.DummyList(), self.DummyList()] self.run_threads(self.multipass, (self.barrier, results, passes)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_barrier_10(self): """ Test that a barrier works for 10 consecutive runs @@ -2216,6 +2257,7 @@ def _test_wait_return_f(cls, barrier, queue): res = barrier.wait() queue.put(res) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_wait_return(self): """ test the return value from barrier.wait @@ -2232,6 +2274,7 @@ def _test_action_f(cls, barrier, results): if len(results) != 1: raise RuntimeError + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_action(self): """ Test the 'action' callback @@ -2254,6 +2297,7 @@ def _test_abort_f(cls, barrier, results1, results2): except RuntimeError: barrier.abort() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_abort(self): """ Test that an abort will put the barrier in a broken state @@ -2284,6 +2328,7 @@ def _test_reset_f(cls, barrier, results1, results2, results3): barrier.wait() results3.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_reset(self): """ Test that a 'reset' on a barrier frees the waiting threads @@ -2319,6 +2364,7 @@ def _test_abort_and_reset_f(cls, barrier, barrier2, barrier.wait() results3.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_abort_and_reset(self): """ Test that a barrier can be reset after being broken. @@ -2345,6 +2391,7 @@ def _test_timeout_f(cls, barrier, results): except threading.BrokenBarrierError: results.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_timeout(self): """ Test wait(timeout) @@ -2364,6 +2411,7 @@ def _test_default_timeout_f(cls, barrier, results): except threading.BrokenBarrierError: results.append(True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_default_timeout(self): """ Test the barrier's default timeout @@ -2385,6 +2433,7 @@ def _test_thousand_f(cls, barrier, passes, conn, lock): with lock: conn.send(i) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_thousand(self): if self.TYPE == 'manager': self.skipTest('test not appropriate for {}'.format(self.TYPE)) @@ -2426,7 +2475,7 @@ def _test(cls, values): for sv, cv in zip(values, cls.codes_values): sv.value = cv[2] - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_value(self, raw=False): if raw: values = [self.RawValue(code, value) @@ -2490,6 +2539,7 @@ def f(cls, seq): for i in range(1, len(seq)): seq[i] += seq[i-1] + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipIf(c_int is None, "requires _ctypes") def test_array(self, raw=False): seq = [680, 626, 934, 821, 150, 233, 548, 982, 714, 831] @@ -2801,8 +2851,16 @@ class _TestPool(BaseTestCase): @classmethod def setUpClass(cls): - super().setUpClass() - cls.pool = cls.Pool(4) + # gh-135427 + # In some of the tests, a forked child forks another child of itself. In that case, using + # warnings_helper.ignore_warnings decorator does not actually ignore the warning from that + # child of child, and a warnings_helper.ignore_warnings exception is raised. + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) + super().setUpClass() + cls.pool = cls.Pool(4) @classmethod def tearDownClass(cls): @@ -2901,6 +2959,7 @@ def test_async(self): self.assertEqual(get(), 49) self.assertTimingAlmostEqual(get.elapsed, TIMEOUT1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_async_timeout(self): p = self.Pool(3) try: @@ -2998,6 +3057,7 @@ def test_imap_unordered_handle_iterable_exception(self): self.assertIn(value, expected_values) expected_values.remove(value) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_make_pool(self): expected_error = (RemoteError if self.TYPE == 'manager' else ValueError) @@ -3013,6 +3073,7 @@ def test_make_pool(self): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_terminate(self): # Simulate slow tasks which take "forever" to complete sleep_time = support.LONG_TIMEOUT @@ -3030,6 +3091,7 @@ def test_terminate(self): p.terminate() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_empty_iterable(self): # See Issue 12157 p = self.Pool(1) @@ -3042,6 +3104,7 @@ def test_empty_iterable(self): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_context(self): if self.TYPE == 'processes': L = list(range(10)) @@ -3056,6 +3119,7 @@ def test_context(self): def _test_traceback(cls): raise RuntimeError(123) # some comment + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_traceback(self): # We want ensure that the traceback from the child process is # contained in the traceback raised in the main process. @@ -3095,9 +3159,11 @@ def test_traceback(self): p.join() @classmethod + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def _test_wrapped_exception(cls): raise RuntimeError('foo') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_wrapped_exception(self): # Issue #20980: Should not wrap exception when using thread pool with self.Pool(1) as p: @@ -3105,6 +3171,7 @@ def test_wrapped_exception(self): p.apply(self._test_wrapped_exception) p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_no_failfast(self): # Issue #23992: the fail-fast behaviour when an exception is raised # during map() would make Pool.join() deadlock, because a worker @@ -3140,6 +3207,7 @@ def test_release_task_refs(self): # they were released too. self.assertEqual(CountedObject.n_instances, 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_enter(self): if self.TYPE == 'manager': self.skipTest("test not applicable to manager") @@ -3156,6 +3224,7 @@ def test_enter(self): pass pool.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_resource_warning(self): if self.TYPE == 'manager': self.skipTest("test not applicable to manager") @@ -3181,6 +3250,7 @@ def unpickleable_result(): class _TestPoolWorkerErrors(BaseTestCase): ALLOWED_TYPES = ('processes', ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_async_error_callback(self): p = multiprocessing.Pool(2) @@ -3196,6 +3266,7 @@ def errback(exc): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_unpickleable_result(self): from multiprocessing.pool import MaybeEncodingError p = multiprocessing.Pool(2) @@ -3221,6 +3292,7 @@ def errback(exc): class _TestPoolWorkerLifetime(BaseTestCase): ALLOWED_TYPES = ('processes', ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_pool_worker_lifetime(self): p = multiprocessing.Pool(3, maxtasksperchild=10) self.assertEqual(3, len(p._pool)) @@ -3250,6 +3322,7 @@ def test_pool_worker_lifetime(self): p.close() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_pool_worker_lifetime_early_close(self): # Issue #10332: closing a pool whose workers have limited lifetimes # before all the tasks completed would make join() hang. @@ -3323,6 +3396,7 @@ class _TestMyManager(BaseTestCase): ALLOWED_TYPES = ('manager',) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_mymanager(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) manager.start() @@ -3334,6 +3408,7 @@ def test_mymanager(self): # which happens on slow buildbots. self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_mymanager_context(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) with manager: @@ -3343,6 +3418,7 @@ def test_mymanager_context(self): # which happens on slow buildbots. self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_mymanager_context_prestarted(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) manager.start() @@ -3413,6 +3489,7 @@ def _putter(cls, address, authkey): # Note that xmlrpclib will deserialize object as a list not a tuple queue.put(tuple(cls.values)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_remote(self): authkey = os.urandom(32) @@ -3454,6 +3531,7 @@ def _putter(cls, address, authkey): queue = manager.get_queue() queue.put('hello world') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_rapid_restart(self): authkey = os.urandom(32) manager = QueueManager( @@ -3506,12 +3584,21 @@ def recv(self): class TestManagerExceptions(unittest.TestCase): # Issue 106558: Manager exceptions avoids creating cyclic references. def setUp(self): - self.mgr = multiprocessing.Manager() + # gh-135427 + # In some of the tests, a forked child forks another child of itself. In that case, using + # warnings_helper.ignore_warnings decorator does not actually ignore the warning from that + # child of child, and a warnings_helper.ignore_warnings exception is raised. + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) + self.mgr = multiprocessing.Manager() def tearDown(self): self.mgr.shutdown() self.mgr.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_queue_get(self): queue = self.mgr.Queue() if gc.isenabled(): @@ -3523,6 +3610,7 @@ def test_queue_get(self): wr = weakref.ref(e) self.assertEqual(wr(), None) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_dispatch(self): if gc.isenabled(): gc.disable() @@ -3549,6 +3637,7 @@ def _echo(cls, conn): conn.send_bytes(msg) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_connection(self): conn, child_conn = self.Pipe() @@ -3641,6 +3730,7 @@ def test_duplex_false(self): self.assertRaises(OSError, writer.recv) self.assertRaises(OSError, writer.poll) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_spawn_close(self): # We test that a pipe connection can be closed by parent # process immediately after child is spawned. On Windows this @@ -3717,6 +3807,7 @@ def _writefd(cls, conn, data, create_dummy_fds=False): os.write(fd, data) os.close(fd) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") def test_fd_transfer(self): if self.TYPE != 'processes': @@ -3736,6 +3827,7 @@ def test_fd_transfer(self): with open(os_helper.TESTFN, "rb") as f: self.assertEqual(f.read(), b"foo") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") @unittest.skipIf(sys.platform == "win32", "test semantics don't make sense on Windows") @@ -3773,6 +3865,7 @@ def test_large_fd_transfer(self): def _send_data_without_fd(self, conn): os.write(conn.fileno(), b"\0") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(HAS_REDUCTION, "test needs multiprocessing.reduction") @unittest.skipIf(sys.platform == "win32", "doesn't make sense on Windows") def test_missing_fd_transfer(self): @@ -3872,6 +3965,7 @@ def _test(cls, address): conn.send('hello') conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_listener_client(self): for family in self.connection.families: l = self.connection.Listener(family=family) @@ -3883,6 +3977,7 @@ def test_listener_client(self): p.join() l.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_issue14725(self): l = self.connection.Listener() p = self.Process(target=self._test, args=(l.address,)) @@ -3928,6 +4023,7 @@ def _child_strings(cls, conn, strings): conn.send_bytes(s) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_strings(self): strings = (b'hello', b'', b'a', b'b', b'', b'bye', b'', b'lop') a, b = self.Pipe() @@ -3951,6 +4047,7 @@ def _child_boundaries(cls, r): # read from it. r.poll(5) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_boundaries(self): r, w = self.Pipe(False) p = self.Process(target=self._child_boundaries, args=(r,)) @@ -3969,6 +4066,7 @@ def _child_dont_merge(cls, b): b.send_bytes(b'b') b.send_bytes(b'cd') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_dont_merge(self): a, b = self.Pipe() self.assertEqual(a.poll(0.0), False) @@ -4037,6 +4135,7 @@ def _remote(cls, conn): conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_pickling(self): families = self.connection.families @@ -4095,6 +4194,7 @@ def child_access(cls, conn): conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_access(self): # On Windows, if we do not specify a destination pid when # using DupHandle then we need to be careful to use the @@ -4258,6 +4358,7 @@ def _double(cls, x, y, z, foo, arr, string): for i in range(len(arr)): arr[i] *= 2 + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_sharedctypes(self, lock=False): x = Value('i', 7, lock=lock) y = Value(c_double, 1.0/3.0, lock=lock) @@ -4519,6 +4620,7 @@ def test_shared_memory_pickle_unpickle_dead_object(self): with self.assertRaises(FileNotFoundError): pickle.loads(pickled_sms) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_shared_memory_across_processes(self): # bpo-40135: don't define shared memory block's name in case of # the failure when we run multiprocessing tests in parallel. @@ -4547,6 +4649,7 @@ def test_shared_memory_across_processes(self): sms.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipIf(os.name != "posix", "not feasible in non-posix platforms") def test_shared_memory_SharedMemoryServer_ignores_sigint(self): # bpo-36368: protect SharedMemoryManager server process from @@ -4592,6 +4695,7 @@ def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # properly released sl. self.assertFalse(err) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_shared_memory_SharedMemoryManager_basics(self): smm1 = multiprocessing.managers.SharedMemoryManager() with self.assertRaises(ValueError): @@ -4931,6 +5035,7 @@ class Foo(object): conn.close() os._exit(0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_finalize(self): conn, child_conn = self.Pipe() @@ -5058,6 +5163,7 @@ def _test_level(cls, conn): logger = multiprocessing.get_logger() conn.send(logger.getEffectiveLevel()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_level(self): LEVEL1 = 32 LEVEL2 = 37 @@ -5142,6 +5248,7 @@ def _killer(cls, pid): time.sleep(0.1) os.kill(pid, signal.SIGUSR1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') def test_poll_eintr(self): got_signal = [False] @@ -5264,14 +5371,23 @@ def initializer(ns): @hashlib_helper.requires_hashdigest('sha256') class TestInitializers(unittest.TestCase): def setUp(self): - self.mgr = multiprocessing.Manager() - self.ns = self.mgr.Namespace() - self.ns.test = 0 + # gh-135427 + # In some of the tests, a forked child forks another child of itself. In that case, using + # warnings_helper.ignore_warnings decorator does not actually ignore the warning from that + # child of child, and a warnings_helper.ignore_warnings exception is raised. + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) + self.mgr = multiprocessing.Manager() + self.ns = self.mgr.Namespace() + self.ns.test = 0 def tearDown(self): self.mgr.shutdown() self.mgr.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_manager_initializer(self): m = multiprocessing.managers.SyncManager() self.assertRaises(TypeError, m.start, 1) @@ -5280,6 +5396,7 @@ def test_manager_initializer(self): m.shutdown() m.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_pool_initializer(self): self.assertRaises(TypeError, multiprocessing.Pool, initializer=1) p = multiprocessing.Pool(1, initializer, (self.ns,)) @@ -5337,16 +5454,19 @@ def flush(self): class TestStdinBadfiledescriptor(unittest.TestCase): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_queue_in_process(self): proc = multiprocessing.Process(target=_test_process) proc.start() proc.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_pool_in_process(self): p = multiprocessing.Process(target=pool_in_process) p.start() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_flushing(self): sio = io.StringIO() flike = _file_like(sio) @@ -5366,6 +5486,7 @@ def _child_test_wait(cls, w, slow): w.send((i, os.getpid())) w.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_wait(self, slow=False): from multiprocessing.connection import wait readers = [] @@ -5406,6 +5527,7 @@ def _child_test_wait_socket(cls, address, slow): s.sendall(('%s\n' % i).encode('ascii')) s.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_wait_socket(self, slow=False): from multiprocessing.connection import wait l = socket.create_server((socket_helper.HOST, 0)) @@ -5470,6 +5592,7 @@ def signal_and_sleep(cls, sem, period): sem.release() time.sleep(period) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_resource('walltime') def test_wait_integer(self): from multiprocessing.connection import wait @@ -5514,6 +5637,7 @@ def test_wait_integer(self): p.terminate() p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_neg_timeout(self): from multiprocessing.connection import wait a, b = multiprocessing.Pipe() @@ -5591,6 +5715,7 @@ def _test_timeout(cls, child, address): conn.send(456) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_timeout(self): old_timeout = socket.getdefaulttimeout() try: @@ -5648,6 +5773,7 @@ def child(cls, n, conn): conn.send(len(util._afterfork_registry)) conn.close() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_lock(self): r, w = multiprocessing.Pipe(False) l = util.ForkAwareThreadLock() @@ -5699,6 +5825,7 @@ def _test_closefds(cls, conn, fd): s.close() conn.send(None) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_closefd(self): if not HAS_REDUCTION: raise unittest.SkipTest('requires fd pickling') @@ -5744,6 +5871,7 @@ def handler(signum, frame): conn.send(x) conn.send_bytes(b'x' * cls.CONN_MAX_SIZE) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') def test_ignore(self): conn, child_conn = multiprocessing.Pipe() @@ -5777,6 +5905,7 @@ def handler(signum, frame): a = l.accept() a.send('welcome') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') def test_ignore_listener(self): conn, child_conn = multiprocessing.Pipe() @@ -5811,6 +5940,7 @@ def check_context(self, ctx): p.join() self.assertEqual(child_method, ctx.get_start_method()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_context(self): for method in ('fork', 'spawn', 'forkserver'): try: @@ -5831,6 +5961,7 @@ def test_context_check_module_types(self): with self.assertRaisesRegex(TypeError, 'module_names must be a list of strings'): ctx.set_forkserver_preload([1, 2, 3]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_set_get(self): multiprocessing.set_forkserver_preload(PRELOAD) count = 0 @@ -5888,6 +6019,7 @@ def test_preload_resources(self): print(err) self.fail("failed spawning forkserver or grandchild") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipIf(sys.platform == "win32", "Only Spawn on windows so no risk of mixing") @only_run_in_spawn_testsuite("avoids redundant testing.") @@ -5921,6 +6053,7 @@ def _put_two_and_nest_once(cls, queue): process.start() process.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_nested_startmethod(self): # gh-108520: Regression test to ensure that child process can send its # arguments to another process @@ -6076,6 +6209,7 @@ def _is_resource_tracker_reused(conn, pid): reused &= _resource_tracker._check_alive() conn.send(reused) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_resource_tracker_reused(self): from multiprocessing.resource_tracker import _resource_tracker _resource_tracker.ensure_running() @@ -6177,6 +6311,7 @@ def test_empty_exceptions(self): with self.assertRaisesRegex(OSError, 'is closed'): q.empty() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() @@ -6284,7 +6419,15 @@ def test_list(self): def setUp(self): self.manager = self.manager_class() - self.manager.start() + # gh-135427 + # In some of the tests, a forked child forks another child of itself. In that case, using + # warnings_helper.ignore_warnings decorator does not actually ignore the warning from that + # child of child, and a warnings_helper.ignore_warnings exception is raised. + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) + self.manager.start() self.proc = None def tearDown(self): @@ -6333,6 +6476,7 @@ def _test_event(cls, obj): obj.clear() obj.wait(0.001) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_event(self): o = self.manager.Event() o.set() @@ -6345,6 +6489,7 @@ def _test_lock(cls, obj): obj.acquire() obj.locked() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_lock(self, lname="Lock"): o = getattr(self.manager, lname)() self.run_worker(self._test_lock, o) @@ -6357,6 +6502,7 @@ def _test_rlock(cls, obj): obj.release() obj.locked() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_rlock(self, lname="RLock"): o = getattr(self.manager, lname)() self.run_worker(self._test_rlock, o) @@ -6365,6 +6511,7 @@ def test_rlock(self, lname="RLock"): def _test_semaphore(cls, obj): obj.acquire() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_semaphore(self, sname="Semaphore"): o = getattr(self.manager, sname)() self.run_worker(self._test_semaphore, o) @@ -6378,6 +6525,7 @@ def _test_condition(cls, obj): obj.acquire() obj.release() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_condition(self): o = self.manager.Condition() self.run_worker(self._test_condition, o) @@ -6387,6 +6535,7 @@ def _test_barrier(cls, obj): assert obj.parties == 5 obj.reset() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_barrier(self): o = self.manager.Barrier(5) self.run_worker(self._test_barrier, o) @@ -6397,6 +6546,7 @@ def _test_pool(cls, obj): with obj: pass + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_pool(self): o = self.manager.Pool(processes=4) self.run_worker(self._test_pool, o) @@ -6411,6 +6561,7 @@ def _test_queue(cls, obj): assert obj.get() == 6 assert obj.empty() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_queue(self, qname="Queue"): o = getattr(self.manager, qname)(2) o.put(5) @@ -6419,6 +6570,7 @@ def test_queue(self, qname="Queue"): assert o.empty() assert not o.full() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_joinable_queue(self): self.test_queue("JoinableQueue") @@ -6453,6 +6605,7 @@ def _test_list(cls, obj): obj.clear() case.assertEqual(len(obj), 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_list(self): o = self.manager.list() o.append(5) @@ -6494,6 +6647,7 @@ def _test_dict(cls, obj): obj.clear() case.assertEqual(len(obj), 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_dict(self): o = self.manager.dict() o['foo'] = 5 @@ -6508,6 +6662,7 @@ def _test_value(cls, obj): case.assertEqual(obj.get(), 1) obj.set(2) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_value(self): o = self.manager.Value('i', 1) self.run_worker(self._test_value, o) @@ -6522,6 +6677,7 @@ def _test_array(cls, obj): case.assertEqual(len(obj), 2) case.assertListEqual(list(obj), [0, 1]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_array(self): o = self.manager.Array('i', [0, 1]) self.run_worker(self._test_array, o) @@ -6532,6 +6688,7 @@ def _test_namespace(cls, obj): case.assertEqual(obj.x, 0) case.assertEqual(obj.y, 1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_namespace(self): o = self.manager.Namespace() o.x = 0 @@ -6651,6 +6808,7 @@ def _test_set_comparisons(cls, obj): case.assertGreater(obj, {'a'}) case.assertGreaterEqual(obj, {'a', 'b'}) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_set(self): o = self.manager.set() self.run_worker(self._test_set_operator_symbols, o) @@ -6668,6 +6826,7 @@ def test_set_init(self): self.assertSetEqual(o, {"a", "b", "c"}) self.assertRaises(RemoteError, self.manager.set, 1234) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_set_contain_all_method(self): o = self.manager.set() set_methods = { @@ -6721,6 +6880,7 @@ def exit_handler(): f.write("deadbeef") atexit.register(exit_handler) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_atexit(self): # gh-83856 with os_helper.temp_dir() as temp_dir: @@ -6873,6 +7033,7 @@ def f(x): return x*x self.assertEqual("332833500", out.decode('utf-8').strip()) self.assertFalse(err, msg=err.decode('utf-8')) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_forked_thread_not_started(self): # gh-134381: Ensure that a thread that has not been started yet in # the parent process can be started within a forked child process. @@ -6974,8 +7135,16 @@ def Pool(cls, *args, **kwds): @classmethod def setUpClass(cls): - super().setUpClass() - cls.manager = multiprocessing.Manager() + # gh-135427 + # In some of the tests, a forked child forks another child of itself. In that case, using + # warnings_helper.ignore_warnings decorator does not actually ignore the warning from that + # child of child, and a warnings_helper.ignore_warnings exception is raised. + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) + super().setUpClass() + cls.manager = multiprocessing.Manager() @classmethod def tearDownClass(cls): @@ -7142,3 +7311,62 @@ class SemLock(_multiprocessing.SemLock): name = f'test_semlock_subclass-{os.getpid()}' s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) + + +@unittest.skipIf(sys.platform != "linux", "Linux only") +class ForkInThreads(unittest.TestCase): + + def test_fork(self): + code = """ + import os, sys, threading, time + + t = threading.Thread(target=time.sleep, args=(1,), daemon=True) + t.start() + + assert threading.active_count() == 2 + + pid = os.fork() + if pid < 0: + print("Fork failed") + elif pid == 0: + print("In child") + sys.exit(0) + print("In parent") + """ + + res = assert_python_ok("-c", code, PYTHONWARNINGS='always') + self.assertIn(b'In child', res.out) + self.assertIn(b'In parent', res.out) + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of fork() may lead to deadlocks in the child', res.err) + + res = assert_python_failure("-c", code, PYTHONWARNINGS='error') + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of fork() may lead to deadlocks in the child', res.err) + + def test_forkpty(self): + code = """ + import os, sys, threading, time + + t = threading.Thread(target=time.sleep, args=(1,), daemon=True) + t.start() + + assert threading.active_count() == 2 + + pid, _ = os.forkpty() + if pid < 0: + print(f"forkpty failed") + elif pid == 0: + print(f"In child") + sys.exit(0) + print(f"In parent") + """ + + res = assert_python_ok("-c", code, PYTHONWARNINGS='always') + self.assertIn(b'In parent', res.out) + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) + + res = assert_python_failure("-c", code, PYTHONWARNINGS='error') + self.assertIn(b'DeprecationWarning', res.err) + self.assertIn(b'is multi-threaded, use of forkpty() may lead to deadlocks in the child', res.err) diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py index 5f6f14afd74a6e..c802f18b79f7f4 100644 --- a/Lib/test/support/warnings_helper.py +++ b/Lib/test/support/warnings_helper.py @@ -5,6 +5,8 @@ import sys import warnings +import inspect + def import_deprecated(name): """Import *name* while suppressing DeprecationWarning.""" @@ -49,13 +51,44 @@ def ignore_warnings(*, category): more noisy and tools like 'git blame' less useful. """ def decorator(test): + if inspect.iscoroutinefunction(test): + @functools.wraps(test) + async def async_wrapper(self, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=category) + return await test(self, *args, **kwargs) + return async_wrapper + else: + @functools.wraps(test) + def wrapper(self, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', category=category) + return test(self, *args, **kwargs) + return wrapper + return decorator + + +def ignore_fork_in_thread_deprecation_warnings(test): + """Decorator to suppress the deprecation warnings related to running a fork within a thread. + """ + if inspect.iscoroutinefunction(test): + @functools.wraps(test) + async def async_wrapper(self, *args, **kwargs): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) + return await test(self, *args, **kwargs) + return async_wrapper + else: @functools.wraps(test) def wrapper(self, *args, **kwargs): with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=category) + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) return test(self, *args, **kwargs) return wrapper - return decorator class WarningsRecorder(object): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 22982dc9d8aefe..73f7350555d6ed 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -15,7 +15,7 @@ from unittest import mock from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper from test.support import socket_helper from test.support import wait_process from test.support import hashlib_helper @@ -1182,6 +1182,7 @@ async def runner(): @support.requires_fork() class TestFork(unittest.IsolatedAsyncioTestCase): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 async def test_fork_not_share_event_loop(self): # The forked process should not share the event loop with the parent loop = asyncio.get_running_loop() @@ -1206,6 +1207,7 @@ async def test_fork_not_share_event_loop(self): self.assertEqual(result, b'NO LOOP') wait_process(pid, exitcode=0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @hashlib_helper.requires_hashdigest('md5') @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_signal_handling(self): @@ -1253,6 +1255,7 @@ async def func(): self.assertFalse(parent_handled.is_set()) self.assertTrue(child_handled.is_set()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @hashlib_helper.requires_hashdigest('md5') @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_run(self): @@ -1273,6 +1276,7 @@ async def child_main(): self.assertEqual(result.value, 42) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @hashlib_helper.requires_hashdigest('md5') @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) def test_fork_asyncio_subprocess(self): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 8830641f0abdc7..8a3a5a584e41a5 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -31,6 +31,7 @@ from test import support from test.support import cpython_only, swap_attr from test.support import async_yield, run_yielding_async_fn +from test.support import warnings_helper from test.support.import_helper import import_module from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok @@ -2545,6 +2546,7 @@ def run_child(self, child, terminal_input): finally: signal.signal(signal.SIGHUP, old_sighup) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def _run_child(self, child, terminal_input): r, w = os.pipe() # Pipe test results from child back to parent try: diff --git a/Lib/test/test_concurrent_futures/executor.py b/Lib/test/test_concurrent_futures/executor.py index 95bf8fcd25bf54..62aa568e86a011 100644 --- a/Lib/test/test_concurrent_futures/executor.py +++ b/Lib/test/test_concurrent_futures/executor.py @@ -5,7 +5,7 @@ from concurrent import futures from operator import add from test import support -from test.support import Py_GIL_DISABLED +from test.support import Py_GIL_DISABLED, warnings_helper def mul(x, y): @@ -43,10 +43,12 @@ class ExecutorTest: # Executor.shutdown() and context manager usage is tested by # ExecutorShutdownTest. + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_submit(self): future = self.executor.submit(pow, 2, 8) self.assertEqual(256, future.result()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_submit_keyword(self): future = self.executor.submit(mul, 2, y=8) self.assertEqual(16, future.result()) @@ -57,6 +59,7 @@ def test_submit_keyword(self): with self.assertRaises(TypeError): self.executor.submit(arg=1) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map(self): self.assertEqual( list(self.executor.map(pow, range(10), range(10))), @@ -66,6 +69,7 @@ def test_map(self): list(self.executor.map(pow, range(10), range(10), chunksize=3)), list(map(pow, range(10), range(10)))) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_exception(self): i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5]) self.assertEqual(i.__next__(), (0, 1)) @@ -73,6 +77,7 @@ def test_map_exception(self): with self.assertRaises(ZeroDivisionError): i.__next__() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_resource('walltime') def test_map_timeout(self): results = [] @@ -108,6 +113,7 @@ def test_map_buffersize_value_validation(self): ): self.executor.map(str, range(4), buffersize=buffersize) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_buffersize(self): ints = range(4) for buffersize in (1, 2, len(ints), len(ints) * 2): @@ -115,6 +121,7 @@ def test_map_buffersize(self): res = self.executor.map(str, ints, buffersize=buffersize) self.assertListEqual(list(res), ["0", "1", "2", "3"]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_buffersize_on_multiple_iterables(self): ints = range(4) for buffersize in (1, 2, len(ints), len(ints) * 2): @@ -122,12 +129,14 @@ def test_map_buffersize_on_multiple_iterables(self): res = self.executor.map(add, ints, ints, buffersize=buffersize) self.assertListEqual(list(res), [0, 2, 4, 6]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_buffersize_on_infinite_iterable(self): res = self.executor.map(str, itertools.count(), buffersize=2) self.assertEqual(next(res, None), "0") self.assertEqual(next(res, None), "1") self.assertEqual(next(res, None), "2") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_buffersize_on_multiple_infinite_iterables(self): res = self.executor.map( add, @@ -147,6 +156,7 @@ def test_map_buffersize_without_iterable(self): res = self.executor.map(str, buffersize=2) self.assertIsNone(next(res, None)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_buffersize_when_buffer_is_full(self): ints = iter(range(4)) buffersize = 2 @@ -158,6 +168,7 @@ def test_map_buffersize_when_buffer_is_full(self): msg="should have fetched only `buffersize` elements from `ints`.", ) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_shutdown_race_issue12456(self): # Issue #12456: race condition at shutdown where trying to post a # sentinel in the call queue blocks (the queue is full while processes @@ -165,6 +176,7 @@ def test_shutdown_race_issue12456(self): self.executor.map(str, [2] * (self.worker_count + 1)) self.executor.shutdown() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.cpython_only def test_no_stale_references(self): # Issue #16284: check that the executors don't unnecessarily hang onto @@ -209,6 +221,7 @@ def test_max_workers_negative(self): "than 0"): self.executor_type(max_workers=number) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_free_reference(self): # Issue #14406: Result iterator should not keep an internal # reference to result objects. @@ -221,6 +234,7 @@ def test_free_reference(self): if wr() is None: break + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_swallows_falsey_exceptions(self): # see gh-132063: Prevent exceptions that evaluate as falsey # from being ignored. diff --git a/Lib/test/test_concurrent_futures/test_as_completed.py b/Lib/test/test_concurrent_futures/test_as_completed.py index c90b0021d85fc7..9f7ecf18bbf0c8 100644 --- a/Lib/test/test_concurrent_futures/test_as_completed.py +++ b/Lib/test/test_concurrent_futures/test_as_completed.py @@ -7,6 +7,7 @@ CANCELLED_AND_NOTIFIED, FINISHED, Future) from test import support +from test.support import warnings_helper from .util import ( PENDING_FUTURE, RUNNING_FUTURE, @@ -19,6 +20,7 @@ def mul(x, y): class AsCompletedTests: + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_no_timeout(self): future1 = self.executor.submit(mul, 2, 21) future2 = self.executor.submit(mul, 7, 6) @@ -35,6 +37,7 @@ def test_no_timeout(self): future1, future2]), completed) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_future_times_out(self): """Test ``futures.as_completed`` timing out before completing it's final future.""" @@ -62,6 +65,7 @@ def test_future_times_out(self): # Check that ``future`` wasn't completed. self.assertEqual(completed_futures, already_completed) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_duplicate_futures(self): # Issue 20367. Duplicate futures should not raise exceptions or give # duplicate responses. diff --git a/Lib/test/test_concurrent_futures/test_deadlock.py b/Lib/test/test_concurrent_futures/test_deadlock.py index a465400509d411..756a68a029b601 100644 --- a/Lib/test/test_concurrent_futures/test_deadlock.py +++ b/Lib/test/test_concurrent_futures/test_deadlock.py @@ -10,6 +10,7 @@ from concurrent.futures.process import BrokenProcessPool, _ThreadWakeup from test import support +from test.support import warnings_helper from .util import ( create_executor_tests, setup_module, @@ -111,6 +112,7 @@ def _fail_on_deadlock(self, executor): print(f"\nTraceback:\n {tb}", file=sys.__stderr__) self.fail(f"Executor deadlock:\n\n{tb}") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def _check_error(self, error, func, *args, ignore_stderr=False): # test for deadlock caused by crashes or exiting in a pool self.executor.shutdown(wait=True) @@ -199,6 +201,7 @@ def test_exit_during_result_unpickle_in_result_handler(self): # the result_handler thread self._check_error(BrokenProcessPool, _return_instance, ExitAtUnpickle) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True) def test_shutdown_deadlock(self): # Test that the pool calling shutdown do not cause deadlock @@ -212,6 +215,7 @@ def test_shutdown_deadlock(self): with self.assertRaises(BrokenProcessPool): f.result() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_shutdown_deadlock_pickle(self): # Test that the pool calling shutdown with wait=False does not cause # a deadlock if a task fails at pickle after the shutdown call. @@ -238,6 +242,7 @@ def test_shutdown_deadlock_pickle(self): # dangling threads executor_manager.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True) def test_crash_big_data(self): # Test that there is a clean exception instead of a deadlock when a @@ -254,6 +259,7 @@ def test_crash_big_data(self): executor.shutdown(wait=True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_gh105829_should_not_deadlock_if_wakeup_pipe_full(self): # Issue #105829: The _ExecutorManagerThread wakeup pipe could # fill up and block. See: https://github.com/python/cpython/issues/105829 diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index 6b8484c0d5f197..2937e80a5c8ed0 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -11,6 +11,7 @@ from logging.handlers import QueueHandler from test import support +from test.support import warnings_helper from .util import ExecutorMixin, create_executor_tests, setup_module @@ -48,6 +49,7 @@ def setUp(self): initargs=('initialized',)) super().setUp() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_initializer(self): futures = [self.executor.submit(get_init_status) for _ in range(self.worker_count)] @@ -74,6 +76,7 @@ def setUp(self): self.executor_kwargs = dict(initializer=init_fail) super().setUp() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_initializer(self): with self._assert_logged('ValueError: error in initializer'): try: diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 3f13a1900a4ca4..b47448de40caa7 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -9,7 +9,7 @@ from concurrent.futures.process import BrokenProcessPool from test import support -from test.support import hashlib_helper +from test.support import hashlib_helper, warnings_helper from test.test_importlib.metadata.fixtures import parameterize from .executor import ExecutorTest, mul @@ -49,6 +49,7 @@ def test_max_workers_too_large(self): "max_workers must be <= 61"): futures.ProcessPoolExecutor(max_workers=62) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_killed_child(self): # When a child process is abruptly terminated, the whole pool gets # "broken". @@ -61,6 +62,7 @@ def test_killed_child(self): # Submitting other jobs fails as well. self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_map_chunksize(self): def bad_map(): list(self.executor.map(pow, range(40), range(40), chunksize=-1)) @@ -81,6 +83,7 @@ def bad_map(): def _test_traceback(cls): raise RuntimeError(123) # some comment + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_traceback(self): # We want ensure that the traceback from the child process is # contained in the traceback raised in the main process. @@ -103,6 +106,7 @@ def test_traceback(self): self.assertIn('raise RuntimeError(123) # some comment', f1.getvalue()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @hashlib_helper.requires_hashdigest('md5') def test_ressources_gced_in_workers(self): # Ensure that argument for a job are correctly gc-ed after the job @@ -123,6 +127,7 @@ def test_ressources_gced_in_workers(self): mgr.shutdown() mgr.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_saturation(self): executor = self.executor mp_context = self.get_context() @@ -208,6 +213,7 @@ def test_max_tasks_early_shutdown(self): for i, future in enumerate(futures): self.assertEqual(future.result(), mul(i, i)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_python_finalization_error(self): # gh-109047: Catch RuntimeError on thread creation # during Python finalization. @@ -258,6 +264,7 @@ def test_force_shutdown_workers_invalid_op(self): executor._force_shutdown, operation='invalid operation'), + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @parameterize(*FORCE_SHUTDOWN_PARAMS) def test_force_shutdown_workers(self, function_name): manager = self.get_context().Manager() diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 99b315b47e2530..854af20cd6b1a8 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -6,6 +6,7 @@ from concurrent import futures from test import support +from test.support import warnings_helper from test.support.script_helper import assert_python_ok from .util import ( @@ -77,12 +78,14 @@ def run_last(): self.assertIn("RuntimeError: cannot schedule new futures", err.decode()) self.assertEqual(out.strip(), b"runtime-error") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_hang_issue12364(self): fs = [self.executor.submit(time.sleep, 0.1) for _ in range(50)] self.executor.shutdown() for f in fs: f.result() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_cancel_futures(self): assert self.worker_count <= 5, "test needs few workers" fs = [self.executor.submit(time.sleep, .1) for _ in range(50)] @@ -128,6 +131,7 @@ def test_hang_gh83386(self): self.assertFalse(err) self.assertEqual(out.strip(), b"apple") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_hang_gh94440(self): """shutdown(wait=True) doesn't hang when a future was submitted and quickly canceled right before shutdown. @@ -171,6 +175,7 @@ def acquire_lock(lock): for t in self.executor._threads: t.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_context_manager_shutdown(self): with futures.ThreadPoolExecutor(max_workers=5) as e: executor = e @@ -180,6 +185,7 @@ def test_context_manager_shutdown(self): for t in executor._threads: t.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_del_shutdown(self): executor = futures.ThreadPoolExecutor(max_workers=5) res = executor.map(abs, range(-5, 5)) @@ -193,6 +199,7 @@ def test_del_shutdown(self): # executor got shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_shutdown_no_wait(self): # Ensure that the executor cleans up the threads when calling # shutdown with wait=False @@ -207,7 +214,7 @@ def test_shutdown_no_wait(self): # executor got shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_thread_names_assigned(self): executor = futures.ThreadPoolExecutor( max_workers=5, thread_name_prefix='SpecialPool') @@ -220,6 +227,7 @@ def test_thread_names_assigned(self): self.assertRegex(t.name, r'^SpecialPool_[0-4]$') t.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_thread_names_default(self): executor = futures.ThreadPoolExecutor(max_workers=5) executor.map(abs, range(-5, 5)) @@ -253,6 +261,7 @@ def test_cancel_futures_wait_false(self): class ProcessPoolShutdownTest(ExecutorShutdownTest): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_processes_terminate(self): def acquire_lock(lock): lock.acquire() @@ -276,6 +285,7 @@ def acquire_lock(lock): for p in processes.values(): p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_context_manager_shutdown(self): with futures.ProcessPoolExecutor( max_workers=5, mp_context=self.get_context()) as e: @@ -286,6 +296,7 @@ def test_context_manager_shutdown(self): for p in processes.values(): p.join() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_del_shutdown(self): executor = futures.ProcessPoolExecutor( max_workers=5, mp_context=self.get_context()) @@ -308,6 +319,7 @@ def test_del_shutdown(self): # executor got shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_shutdown_no_wait(self): # Ensure that the executor cleans up the processes when calling # shutdown with wait=False diff --git a/Lib/test/test_concurrent_futures/test_wait.py b/Lib/test/test_concurrent_futures/test_wait.py index cc387883141b0e..9388d3f19bb6ee 100644 --- a/Lib/test/test_concurrent_futures/test_wait.py +++ b/Lib/test/test_concurrent_futures/test_wait.py @@ -3,7 +3,7 @@ import unittest from concurrent import futures from test import support -from test.support import threading_helper +from test.support import threading_helper, warnings_helper from .util import ( CANCELLED_FUTURE, CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, @@ -22,6 +22,7 @@ def wait_and_raise(e): class WaitTests: + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_20369(self): # See https://bugs.python.org/issue20369 future = self.executor.submit(mul, 1, 2) @@ -30,7 +31,7 @@ def test_20369(self): self.assertEqual({future}, done) self.assertEqual(set(), not_done) - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_first_completed(self): event = self.create_event() future1 = self.executor.submit(mul, 21, 2) @@ -47,6 +48,7 @@ def test_first_completed(self): event.set() future2.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_first_completed_some_already_completed(self): event = self.create_event() future1 = self.executor.submit(event.wait) @@ -64,6 +66,7 @@ def test_first_completed_some_already_completed(self): event.set() future1.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_first_exception(self): event1 = self.create_event() event2 = self.create_event() @@ -93,6 +96,7 @@ def wait_for_future1(): event2.set() future3.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_first_exception_some_already_complete(self): event = self.create_event() future1 = self.executor.submit(divmod, 21, 0) @@ -114,6 +118,7 @@ def test_first_exception_some_already_complete(self): event.set() future2.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_first_exception_one_already_failed(self): event = self.create_event() future1 = self.executor.submit(event.wait) @@ -129,6 +134,7 @@ def test_first_exception_one_already_failed(self): event.set() future1.result() # wait for job to finish + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_all_completed(self): future1 = self.executor.submit(divmod, 2, 0) future2 = self.executor.submit(mul, 2, 21) @@ -148,6 +154,7 @@ def test_all_completed(self): future2]), finished) self.assertEqual(set(), pending) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_timeout(self): short_timeout = 0.050 diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index b12940414d9142..6e63aa1307cdc8 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -11,6 +11,7 @@ from test import support from test.support import threading_helper +import warnings def create_future(state=PENDING, exception=None, result=None): @@ -51,7 +52,15 @@ def setUp(self): max_workers=self.worker_count, mp_context=self.get_context(), **self.executor_kwargs) - self.manager = self.get_context().Manager() + # gh-135427 + # In some of the tests, a forked child forks another child of itself. In that case, using + # warnings_helper.ignore_warnings decorator does not actually ignore the warning from that + # child of child, and a warnings_helper.ignore_warnings exception is raised. + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', + message=".*fork.*may lead to deadlocks in the child.*", + category=DeprecationWarning) + self.manager = self.get_context().Manager() else: self.executor = self.executor_type( max_workers=self.worker_count, diff --git a/Lib/test/test_fork1.py b/Lib/test/test_fork1.py index a6523bbc518176..0d34ae7aa27028 100644 --- a/Lib/test/test_fork1.py +++ b/Lib/test/test_fork1.py @@ -11,6 +11,7 @@ from test.fork_wait import ForkWait from test import support +from test.support import warnings_helper # Skip test if fork does not exist. @@ -19,6 +20,7 @@ class ForkTest(ForkWait): + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_threaded_import_lock_fork(self): """Check fork() in main thread works while a subthread is doing an import""" import_started = threading.Event() @@ -61,7 +63,7 @@ def importer(): except OSError: pass - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_nested_import_lock_fork(self): """Check fork() in main thread works while the main thread is doing an import""" exitcode = 42 diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index e94edcbc107ba9..77431fdac08f23 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -9,6 +9,8 @@ import time import unittest +from test.support import warnings_helper + if not hasattr(select, "kqueue"): raise unittest.SkipTest("test works only on BSD") @@ -257,6 +259,7 @@ def test_fd_non_inheritable(self): self.addCleanup(kqueue.close) self.assertEqual(os.get_inheritable(kqueue.fileno()), False) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_fork() def test_fork(self): # gh-110395: kqueue objects must be closed after fork diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 275f7ce47d09b5..dea2d5c8b18cf4 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -730,6 +730,7 @@ def remove_loop(fname, tries): # based on os.fork existing because that is what users and this test use. # This helps ensure that when fork exists (the important concept) that the # register_at_fork mechanism is also present and used. + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_fork() @threading_helper.requires_working_threading() @skip_if_asan_fork @@ -4045,6 +4046,7 @@ def test_config_queue_handler_invalid_config_does_not_create_multiprocessing_man self._apply_simple_queue_listener_configuration(qspec) manager.assert_not_called() + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @skip_if_tsan_fork @support.requires_subprocess() @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" @@ -4067,6 +4069,7 @@ def test_config_reject_simple_queue_handler_multiprocessing_context(self): with self.assertRaises(ValueError): self._apply_simple_queue_listener_configuration(qspec) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @skip_if_tsan_fork @support.requires_subprocess() @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" @@ -4107,6 +4110,7 @@ def _mpinit_issue121723(qspec, message_to_log): # log a message (this creates a record put in the queue) logging.getLogger().info(message_to_log) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): @@ -5337,6 +5341,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @skip_if_tsan_fork def test_multiprocessing(self): support.skip_if_broken_multiprocessing_synchronize() diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 0169948e453438..6f3156ac93ceec 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -8,7 +8,7 @@ import io import tempfile from test import support -from test.support import import_helper +from test.support import import_helper, warnings_helper from test.support import os_helper from test.support import refleak_helper from test.support import socket_helper @@ -1212,6 +1212,7 @@ def test_add_and_close(self): self.assertEqual(contents, f.read()) self._box = self._factory(self._path) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_fork() @unittest.skipUnless(hasattr(socket, 'socketpair'), "Test needs socketpair().") def test_lock_conflict(self): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index de3a17fe893170..4656373d075058 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3518,6 +3518,7 @@ def test_getppid(self): self.assertEqual(error, b'') self.assertEqual(int(stdout), os.getpid()) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def check_waitpid(self, code, exitcode, callback=None): if sys.platform == 'win32': # On Windows, os.spawnv() simply joins arguments with spaces: @@ -3620,30 +3621,35 @@ def create_args(self, *, with_env=False, use_bytes=False): return program, args + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnl') def test_spawnl(self): program, args = self.create_args() exitcode = os.spawnl(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnle') def test_spawnle(self): program, args = self.create_args(with_env=True) exitcode = os.spawnle(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnlp') def test_spawnlp(self): program, args = self.create_args() exitcode = os.spawnlp(os.P_WAIT, program, *args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnlpe') def test_spawnlpe(self): program, args = self.create_args(with_env=True) exitcode = os.spawnlpe(os.P_WAIT, program, *args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnv') def test_spawnv(self): program, args = self.create_args() @@ -3654,30 +3660,35 @@ def test_spawnv(self): exitcode = os.spawnv(os.P_WAIT, FakePath(program), args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnve') def test_spawnve(self): program, args = self.create_args(with_env=True) exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnvp') def test_spawnvp(self): program, args = self.create_args() exitcode = os.spawnvp(os.P_WAIT, program, args) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnvpe') def test_spawnvpe(self): program, args = self.create_args(with_env=True) exitcode = os.spawnvpe(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnv') def test_nowait(self): program, args = self.create_args() pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -3685,18 +3696,21 @@ def test_spawnve_bytes(self): exitcode = os.spawnve(os.P_WAIT, program, args, self.env) self.assertEqual(exitcode, self.exitcode) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnl') def test_spawnl_noargs(self): program, __ = self.create_args() self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program) self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, program, '') + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnle') def test_spawnle_noargs(self): program, __ = self.create_args() self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, {}) self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, program, '', {}) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnv') def test_spawnv_noargs(self): program, __ = self.create_args() @@ -3705,6 +3719,7 @@ def test_spawnv_noargs(self): self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ('',)) self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, program, ['']) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnve') def test_spawnve_noargs(self): program, __ = self.create_args() @@ -3761,10 +3776,12 @@ def _test_invalid_env(self, spawn): exitcode = spawn(os.P_WAIT, program, args, newenv) self.assertEqual(exitcode, 0) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnve') def test_spawnve_invalid_env(self): self._test_invalid_env(os.spawnve) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_os_func('spawnvpe') def test_spawnvpe_invalid_env(self): self._test_invalid_env(os.spawnvpe) @@ -4881,6 +4898,7 @@ def test_posix_pty_functions(self): self.addCleanup(os.close, son_fd) self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") @support.requires_subprocess() def test_pipe_spawnl(self): diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 479649053abc01..78327bae98cf26 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -11,7 +11,7 @@ from unittest import mock from test import support -from test.support import os_helper +from test.support import os_helper, warnings_helper try: # Some of the iOS tests need ctypes to operate. @@ -465,7 +465,7 @@ def test_mac_ver(self): else: self.assertEqual(res[2], 'PowerPC') - + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(sys.platform == 'darwin', "OSX only test") def test_mac_ver_with_fork(self): # Issue7895: platform.mac_ver() crashes when using fork without exec diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index ed4fe8a140879d..d1e2c4582c3d73 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,6 +1,6 @@ import unittest from test.support import ( - is_android, is_apple_mobile, is_wasm32, reap_children, verbose + is_android, is_apple_mobile, is_wasm32, reap_children, verbose, warnings_helper ) from test.support.import_helper import import_module from test.support.os_helper import TESTFN, unlink @@ -194,6 +194,7 @@ def test_openpty(self): s2 = _readline(master_fd) self.assertEqual(b'For my pet fish, Eric.\n', normalize_output(s2)) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_fork(self): debug("calling pty.fork()") pid, master_fd = pty.fork() @@ -295,6 +296,7 @@ def test_master_read(self): self.assertEqual(data, b"") + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 def test_spawn_doesnt_hang(self): self.addCleanup(unlink, TESTFN) with open(TESTFN, 'wb') as f: diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 0217ebd132b110..460695b131bf37 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -14,6 +14,8 @@ from fractions import Fraction from collections import abc, Counter +from test.support import warnings_helper + class MyIndex: def __init__(self, value): @@ -1399,6 +1401,7 @@ def test__all__(self): # tests validity but not completeness of the __all__ list self.assertTrue(set(random.__all__) <= set(dir(random))) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @test.support.requires_fork() def test_after_fork(self): # Test the global Random instance gets reseeded in child diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 0f62f9eb200e42..4e8714c79fcd42 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -17,6 +17,7 @@ from test.support import os_helper from test.support import socket_helper from test.support import threading_helper +from test.support import warnings_helper test.support.requires("network") @@ -43,6 +44,7 @@ def receive(sock, n, timeout=test.support.SHORT_TIMEOUT): raise RuntimeError("timed out on %r" % (sock,)) +@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @test.support.requires_fork() @contextlib.contextmanager def simple_subprocess(testcase): @@ -173,6 +175,7 @@ def test_ThreadingTCPServer(self): socketserver.StreamRequestHandler, self.stream_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_forking def test_ForkingTCPServer(self): with simple_subprocess(self): @@ -192,6 +195,7 @@ def test_ThreadingUnixStreamServer(self): socketserver.StreamRequestHandler, self.stream_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_unix_sockets @requires_forking def test_ForkingUnixStreamServer(self): @@ -210,6 +214,7 @@ def test_ThreadingUDPServer(self): socketserver.DatagramRequestHandler, self.dgram_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_forking def test_ForkingUDPServer(self): with simple_subprocess(self): @@ -229,6 +234,7 @@ def test_ThreadingUnixDatagramServer(self): socketserver.DatagramRequestHandler, self.dgram_examine) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_unix_sockets @requires_forking def test_ForkingUnixDatagramServer(self): @@ -314,11 +320,13 @@ def test_threading_not_handled(self): self.assertIs(cm.exc_type, SystemExit) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_forking def test_forking_handled(self): ForkingErrorTestServer(ValueError) self.check_result(handled=True) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @requires_forking def test_forking_not_handled(self): ForkingErrorTestServer(SystemExit) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 9ec382afb65fe4..f88575140a451b 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -485,6 +485,7 @@ def test_check__all__(self): self.assertRaises(AssertionError, support.check__all__, self, unittest) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @unittest.skipUnless(hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG'), 'need os.waitpid() and os.WNOHANG') @support.requires_fork() diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 6dc6880ff8c356..98fd20935cc201 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -8,7 +8,7 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, interpreter_requires_environment) from test import support -from test.support import force_not_colorized +from test.support import force_not_colorized, warnings_helper from test.support import os_helper from test.support import threading_helper @@ -354,6 +354,7 @@ def fork_child(self): # everything is fine return 0 + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_fork() def test_fork(self): # check that tracemalloc is still working after fork diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 0e1a723ce3a151..b99940556ea6b6 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -13,7 +13,7 @@ from unittest import mock from test import support -from test.support import import_helper +from test.support import import_helper, warnings_helper from test.support.script_helper import assert_python_ok py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) @@ -1112,6 +1112,7 @@ def test_uuid8_uniqueness(self): versions = {u.version for u in uuids} self.assertSetEqual(versions, {8}) + @warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427 @support.requires_fork() def testIssue8621(self): # On at least some versions of OSX self.uuid.uuid4 generates diff --git a/Misc/NEWS.d/next/Library/2025-07-19-11-53-19.gh-issue-135427.iJM_X2.rst b/Misc/NEWS.d/next/Library/2025-07-19-11-53-19.gh-issue-135427.iJM_X2.rst new file mode 100644 index 00000000000000..4e0999e2635ba9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-19-11-53-19.gh-issue-135427.iJM_X2.rst @@ -0,0 +1 @@ +A DeprecationWarning is now provided for os.fork and os.forkpty when run within a thread with -Werror. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 77622fbc4e8065..c04fe319d117e5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7992,7 +7992,7 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // // This should only be called from the parent process after // PyOS_AfterFork_Parent(). -static void +static int warn_about_fork_with_threads(const char* name) { // It's not safe to issue the warning while the world is stopped, because @@ -8043,14 +8043,14 @@ warn_about_fork_with_threads(const char* name) PyObject *threading = PyImport_GetModule(&_Py_ID(threading)); if (!threading) { PyErr_Clear(); - return; + return 0; } PyObject *threading_active = PyObject_GetAttr(threading, &_Py_ID(_active)); if (!threading_active) { PyErr_Clear(); Py_DECREF(threading); - return; + return 0; } PyObject *threading_limbo = PyObject_GetAttr(threading, &_Py_ID(_limbo)); @@ -8058,7 +8058,7 @@ warn_about_fork_with_threads(const char* name) PyErr_Clear(); Py_DECREF(threading); Py_DECREF(threading_active); - return; + return 0; } Py_DECREF(threading); // Duplicating what threading.active_count() does but without holding @@ -8074,7 +8074,7 @@ warn_about_fork_with_threads(const char* name) Py_DECREF(threading_limbo); } if (num_python_threads > 1) { - PyErr_WarnFormat( + return PyErr_WarnFormat( PyExc_DeprecationWarning, 1, #ifdef HAVE_GETPID "This process (pid=%d) is multi-threaded, " @@ -8086,8 +8086,8 @@ warn_about_fork_with_threads(const char* name) getpid(), #endif name); - PyErr_Clear(); } + return 0; } #endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK @@ -8126,7 +8126,8 @@ os_fork1_impl(PyObject *module) /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("fork1"); + if (warn_about_fork_with_threads("fork1") < 0) + return NULL; } if (pid == -1) { errno = saved_errno; @@ -8175,7 +8176,8 @@ os_fork_impl(PyObject *module) /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("fork"); + if (warn_about_fork_with_threads("fork") < 0) + return NULL; } if (pid == -1) { errno = saved_errno; @@ -9030,7 +9032,8 @@ os_forkpty_impl(PyObject *module) /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("forkpty"); + if (warn_about_fork_with_threads("forkpty") < 0) + return NULL; } if (pid == -1) { return posix_error();
Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.
Alternative Proxies: