Skip to content

gh-135427: Fix DeprecationWarning for os.fork when run in threads with -Werror #136796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 24 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
6622c55
gh-135427: Fix DeprecationWarning for os.fork when run in threads wit…
rani-pinchuk Jul 19, 2025
7b13bdc
Merge branch 'main' into fix-issue-135427
rani-pinchuk Jul 19, 2025
bf97fe0
📜🤖 Added by blurb_it.
blurb-it[bot] Jul 19, 2025
3482918
gh-135427: check that exception occured, and also clear later the exc…
rani-pinchuk Jul 19, 2025
ce90d77
Merge remote-tracking branch 'refs/remotes/origin/fix-issue-135427' i…
rani-pinchuk Jul 19, 2025
1a20748
Behavior changed to raise an exception when fork is used within a thr…
rani-pinchuk Jul 20, 2025
9bead0e
Seems like test_multiprocessing fork deprecation warnings are suppres…
rani-pinchuk Jul 20, 2025
d5336dc
Warning of deprecated fork are suppressed in more tests.
rani-pinchuk Jul 20, 2025
ae3a5ed
WIP - suppressing warnings in tests that fail in the tests running on…
rani-pinchuk Jul 20, 2025
f768f5d
WIP - continuing suppressing warnings in tests.
rani-pinchuk Jul 21, 2025
6308e19
WIP - fix imports.
rani-pinchuk Jul 21, 2025
8955136
WIP - more suppressing of warnings
rani-pinchuk Jul 21, 2025
8731cba
WIP - fix ignore_warnings to wrap also async functions
rani-pinchuk Jul 21, 2025
41e8337
WIP - more suppress of warnings.
rani-pinchuk Jul 21, 2025
359b0ed
WIP - more suppress of warnings.
rani-pinchuk Jul 21, 2025
5277d4c
WIP - test.test_multiprocessing_fork gives no errors on local ubuntu …
rani-pinchuk Jul 21, 2025
204c642
WIP - more suppression of warnings
rani-pinchuk Jul 22, 2025
1b44537
WIP - adding more suppressing of the DeprecatinoWarning
rani-pinchuk Jul 22, 2025
c43b3c0
WIP - more suppress of warnings.
rani-pinchuk Jul 22, 2025
5d7ae0c
Merge branch 'main' into fix-issue-135427
rani-pinchuk Jul 22, 2025
868ffd6
WIP - more suppress of warnings.
rani-pinchuk Jul 22, 2025
e1398d8
Add dedicated decorator to ignore only the fork in thread deprecation…
rani-pinchuk Jul 24, 2025
b58b2ca
Replace the generic decorators to ignore deprecation errors with spec…
rani-pinchuk Jul 24, 2025
8d832ac
Linting issue
rani-pinchuk Jul 24, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
250 changes: 239 additions & 11 deletions Lib/test/_test_multiprocessing.py

Large diffs are not rendered by default.

37 changes: 35 additions & 2 deletions Lib/test/support/warnings_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
import sys
import warnings

import inspect


def import_deprecated(name):
"""Import *name* while suppressing DeprecationWarning."""
Expand Down Expand Up @@ -49,13 +51,44 @@ def ignore_warnings(*, category):
more noisy and tools like 'git blame' less useful.
"""
def decorator(test):
if inspect.iscoroutinefunction(test):
@functools.wraps(test)
async def async_wrapper(self, *args, **kwargs):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=category)
return await test(self, *args, **kwargs)
return async_wrapper
else:
@functools.wraps(test)
def wrapper(self, *args, **kwargs):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=category)
return test(self, *args, **kwargs)
return wrapper
return decorator


def ignore_fork_in_thread_deprecation_warnings(test):
"""Decorator to suppress the deprecation warnings related to running a fork within a thread.
"""
if inspect.iscoroutinefunction(test):
@functools.wraps(test)
async def async_wrapper(self, *args, **kwargs):
with warnings.catch_warnings():
warnings.filterwarnings('ignore',
message=".*fork.*may lead to deadlocks in the child.*",
category=DeprecationWarning)
return await test(self, *args, **kwargs)
return async_wrapper
else:
@functools.wraps(test)
def wrapper(self, *args, **kwargs):
with warnings.catch_warnings():
warnings.simplefilter('ignore', category=category)
warnings.filterwarnings('ignore',
message=".*fork.*may lead to deadlocks in the child.*",
category=DeprecationWarning)
return test(self, *args, **kwargs)
return wrapper
return decorator


class WarningsRecorder(object):
Expand Down
6 changes: 5 additions & 1 deletion Lib/test/test_asyncio/test_unix_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from unittest import mock

from test import support
from test.support import os_helper
from test.support import os_helper, warnings_helper
from test.support import socket_helper
from test.support import wait_process
from test.support import hashlib_helper
Expand Down Expand Up @@ -1182,6 +1182,7 @@ async def runner():
@support.requires_fork()
class TestFork(unittest.IsolatedAsyncioTestCase):

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
async def test_fork_not_share_event_loop(self):
# The forked process should not share the event loop with the parent
loop = asyncio.get_running_loop()
Expand All @@ -1206,6 +1207,7 @@ async def test_fork_not_share_event_loop(self):
self.assertEqual(result, b'NO LOOP')
wait_process(pid, exitcode=0)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@hashlib_helper.requires_hashdigest('md5')
@support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True)
def test_fork_signal_handling(self):
Expand Down Expand Up @@ -1253,6 +1255,7 @@ async def func():
self.assertFalse(parent_handled.is_set())
self.assertTrue(child_handled.is_set())

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@hashlib_helper.requires_hashdigest('md5')
@support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True)
def test_fork_asyncio_run(self):
Expand All @@ -1273,6 +1276,7 @@ async def child_main():

self.assertEqual(result.value, 42)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@hashlib_helper.requires_hashdigest('md5')
@support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True)
def test_fork_asyncio_subprocess(self):
Expand Down
2 changes: 2 additions & 0 deletions Lib/test/test_builtin.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from test import support
from test.support import cpython_only, swap_attr
from test.support import async_yield, run_yielding_async_fn
from test.support import warnings_helper
from test.support.import_helper import import_module
from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink)
from test.support.script_helper import assert_python_ok
Expand Down Expand Up @@ -2545,6 +2546,7 @@ def run_child(self, child, terminal_input):
finally:
signal.signal(signal.SIGHUP, old_sighup)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def _run_child(self, child, terminal_input):
r, w = os.pipe() # Pipe test results from child back to parent
try:
Expand Down
16 changes: 15 additions & 1 deletion Lib/test/test_concurrent_futures/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from concurrent import futures
from operator import add
from test import support
from test.support import Py_GIL_DISABLED
from test.support import Py_GIL_DISABLED, warnings_helper


def mul(x, y):
Expand Down Expand Up @@ -43,10 +43,12 @@ class ExecutorTest:

# Executor.shutdown() and context manager usage is tested by
# ExecutorShutdownTest.
@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_submit(self):
future = self.executor.submit(pow, 2, 8)
self.assertEqual(256, future.result())

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_submit_keyword(self):
future = self.executor.submit(mul, 2, y=8)
self.assertEqual(16, future.result())
Expand All @@ -57,6 +59,7 @@ def test_submit_keyword(self):
with self.assertRaises(TypeError):
self.executor.submit(arg=1)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map(self):
self.assertEqual(
list(self.executor.map(pow, range(10), range(10))),
Expand All @@ -66,13 +69,15 @@ def test_map(self):
list(self.executor.map(pow, range(10), range(10), chunksize=3)),
list(map(pow, range(10), range(10))))

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map_exception(self):
i = self.executor.map(divmod, [1, 1, 1, 1], [2, 3, 0, 5])
self.assertEqual(i.__next__(), (0, 1))
self.assertEqual(i.__next__(), (0, 1))
with self.assertRaises(ZeroDivisionError):
i.__next__()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@support.requires_resource('walltime')
def test_map_timeout(self):
results = []
Expand Down Expand Up @@ -108,26 +113,30 @@ def test_map_buffersize_value_validation(self):
):
self.executor.map(str, range(4), buffersize=buffersize)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map_buffersize(self):
ints = range(4)
for buffersize in (1, 2, len(ints), len(ints) * 2):
with self.subTest(buffersize=buffersize):
res = self.executor.map(str, ints, buffersize=buffersize)
self.assertListEqual(list(res), ["0", "1", "2", "3"])

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map_buffersize_on_multiple_iterables(self):
ints = range(4)
for buffersize in (1, 2, len(ints), len(ints) * 2):
with self.subTest(buffersize=buffersize):
res = self.executor.map(add, ints, ints, buffersize=buffersize)
self.assertListEqual(list(res), [0, 2, 4, 6])

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map_buffersize_on_infinite_iterable(self):
res = self.executor.map(str, itertools.count(), buffersize=2)
self.assertEqual(next(res, None), "0")
self.assertEqual(next(res, None), "1")
self.assertEqual(next(res, None), "2")

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map_buffersize_on_multiple_infinite_iterables(self):
res = self.executor.map(
add,
Expand All @@ -147,6 +156,7 @@ def test_map_buffersize_without_iterable(self):
res = self.executor.map(str, buffersize=2)
self.assertIsNone(next(res, None))

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map_buffersize_when_buffer_is_full(self):
ints = iter(range(4))
buffersize = 2
Expand All @@ -158,13 +168,15 @@ def test_map_buffersize_when_buffer_is_full(self):
msg="should have fetched only `buffersize` elements from `ints`.",
)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_shutdown_race_issue12456(self):
# Issue #12456: race condition at shutdown where trying to post a
# sentinel in the call queue blocks (the queue is full while processes
# have exited).
self.executor.map(str, [2] * (self.worker_count + 1))
self.executor.shutdown()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@support.cpython_only
def test_no_stale_references(self):
# Issue #16284: check that the executors don't unnecessarily hang onto
Expand Down Expand Up @@ -209,6 +221,7 @@ def test_max_workers_negative(self):
"than 0"):
self.executor_type(max_workers=number)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_free_reference(self):
# Issue #14406: Result iterator should not keep an internal
# reference to result objects.
Expand All @@ -221,6 +234,7 @@ def test_free_reference(self):
if wr() is None:
break

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_swallows_falsey_exceptions(self):
# see gh-132063: Prevent exceptions that evaluate as falsey
# from being ignored.
Expand Down
4 changes: 4 additions & 0 deletions Lib/test/test_concurrent_futures/test_as_completed.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
CANCELLED_AND_NOTIFIED, FINISHED, Future)

from test import support
from test.support import warnings_helper

from .util import (
PENDING_FUTURE, RUNNING_FUTURE,
Expand All @@ -19,6 +20,7 @@ def mul(x, y):


class AsCompletedTests:
@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_no_timeout(self):
future1 = self.executor.submit(mul, 2, 21)
future2 = self.executor.submit(mul, 7, 6)
Expand All @@ -35,6 +37,7 @@ def test_no_timeout(self):
future1, future2]),
completed)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_future_times_out(self):
"""Test ``futures.as_completed`` timing out before
completing it's final future."""
Expand Down Expand Up @@ -62,6 +65,7 @@ def test_future_times_out(self):
# Check that ``future`` wasn't completed.
self.assertEqual(completed_futures, already_completed)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_duplicate_futures(self):
# Issue 20367. Duplicate futures should not raise exceptions or give
# duplicate responses.
Expand Down
6 changes: 6 additions & 0 deletions Lib/test/test_concurrent_futures/test_deadlock.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from concurrent.futures.process import BrokenProcessPool, _ThreadWakeup

from test import support
from test.support import warnings_helper

from .util import (
create_executor_tests, setup_module,
Expand Down Expand Up @@ -111,6 +112,7 @@ def _fail_on_deadlock(self, executor):
print(f"\nTraceback:\n {tb}", file=sys.__stderr__)
self.fail(f"Executor deadlock:\n\n{tb}")

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def _check_error(self, error, func, *args, ignore_stderr=False):
# test for deadlock caused by crashes or exiting in a pool
self.executor.shutdown(wait=True)
Expand Down Expand Up @@ -199,6 +201,7 @@ def test_exit_during_result_unpickle_in_result_handler(self):
# the result_handler thread
self._check_error(BrokenProcessPool, _return_instance, ExitAtUnpickle)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True)
def test_shutdown_deadlock(self):
# Test that the pool calling shutdown do not cause deadlock
Expand All @@ -212,6 +215,7 @@ def test_shutdown_deadlock(self):
with self.assertRaises(BrokenProcessPool):
f.result()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_shutdown_deadlock_pickle(self):
# Test that the pool calling shutdown with wait=False does not cause
# a deadlock if a task fails at pickle after the shutdown call.
Expand All @@ -238,6 +242,7 @@ def test_shutdown_deadlock_pickle(self):
# dangling threads
executor_manager.join()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@support.skip_if_sanitizer("UBSan: explicit SIGSEV not allowed", ub=True)
def test_crash_big_data(self):
# Test that there is a clean exception instead of a deadlock when a
Expand All @@ -254,6 +259,7 @@ def test_crash_big_data(self):

executor.shutdown(wait=True)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_gh105829_should_not_deadlock_if_wakeup_pipe_full(self):
# Issue #105829: The _ExecutorManagerThread wakeup pipe could
# fill up and block. See: https://github.com/python/cpython/issues/105829
Expand Down
3 changes: 3 additions & 0 deletions Lib/test/test_concurrent_futures/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from logging.handlers import QueueHandler

from test import support
from test.support import warnings_helper

from .util import ExecutorMixin, create_executor_tests, setup_module

Expand Down Expand Up @@ -48,6 +49,7 @@ def setUp(self):
initargs=('initialized',))
super().setUp()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_initializer(self):
futures = [self.executor.submit(get_init_status)
for _ in range(self.worker_count)]
Expand All @@ -74,6 +76,7 @@ def setUp(self):
self.executor_kwargs = dict(initializer=init_fail)
super().setUp()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_initializer(self):
with self._assert_logged('ValueError: error in initializer'):
try:
Expand Down
9 changes: 8 additions & 1 deletion Lib/test/test_concurrent_futures/test_process_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
from concurrent.futures.process import BrokenProcessPool

from test import support
from test.support import hashlib_helper
from test.support import hashlib_helper, warnings_helper
from test.test_importlib.metadata.fixtures import parameterize

from .executor import ExecutorTest, mul
Expand Down Expand Up @@ -49,6 +49,7 @@ def test_max_workers_too_large(self):
"max_workers must be <= 61"):
futures.ProcessPoolExecutor(max_workers=62)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_killed_child(self):
# When a child process is abruptly terminated, the whole pool gets
# "broken".
Expand All @@ -61,6 +62,7 @@ def test_killed_child(self):
# Submitting other jobs fails as well.
self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8)

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_map_chunksize(self):
def bad_map():
list(self.executor.map(pow, range(40), range(40), chunksize=-1))
Expand All @@ -81,6 +83,7 @@ def bad_map():
def _test_traceback(cls):
raise RuntimeError(123) # some comment

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_traceback(self):
# We want ensure that the traceback from the child process is
# contained in the traceback raised in the main process.
Expand All @@ -103,6 +106,7 @@ def test_traceback(self):
self.assertIn('raise RuntimeError(123) # some comment',
f1.getvalue())

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@hashlib_helper.requires_hashdigest('md5')
def test_ressources_gced_in_workers(self):
# Ensure that argument for a job are correctly gc-ed after the job
Expand All @@ -123,6 +127,7 @@ def test_ressources_gced_in_workers(self):
mgr.shutdown()
mgr.join()

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_saturation(self):
executor = self.executor
mp_context = self.get_context()
Expand Down Expand Up @@ -208,6 +213,7 @@ def test_max_tasks_early_shutdown(self):
for i, future in enumerate(futures):
self.assertEqual(future.result(), mul(i, i))

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
def test_python_finalization_error(self):
# gh-109047: Catch RuntimeError on thread creation
# during Python finalization.
Expand Down Expand Up @@ -258,6 +264,7 @@ def test_force_shutdown_workers_invalid_op(self):
executor._force_shutdown,
operation='invalid operation'),

@warnings_helper.ignore_fork_in_thread_deprecation_warnings # gh-135427
@parameterize(*FORCE_SHUTDOWN_PARAMS)
def test_force_shutdown_workers(self, function_name):
manager = self.get_context().Manager()
Expand Down
Loading
Loading
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy