Skip to content

Commit 3664a79

Browse files
pitroupitrou
committed
GH-109978: Allow multiprocessing finalizers to run on a separate thread
1 parent de2a403 commit 3664a79

File tree

15 files changed

+265
-73
lines changed

15 files changed

+265
-73
lines changed

‎Lib/concurrent/futures/process.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -561,9 +561,9 @@ def shutdown_workers(self):
561561
except queue.Full:
562562
break
563563

564-
def join_executor_internals(self):
564+
def join_executor_internals(self, broken=False):
565565
with self.shutdown_lock:
566-
self._join_executor_internals()
566+
self._join_executor_internals(broken)
567567

568568
def _join_executor_internals(self, broken=False):
569569
# If broken, call_queue was closed and so can no longer be used.
@@ -759,7 +759,11 @@ def _start_executor_manager_thread(self):
759759
if not self._safe_to_dynamically_spawn_children: # ie, using fork.
760760
self._launch_processes()
761761
self._executor_manager_thread = _ExecutorManagerThread(self)
762-
self._executor_manager_thread.start()
762+
try:
763+
self._executor_manager_thread.start()
764+
except RuntimeError:
765+
self._broken = "Executor manager thread could not be started"
766+
raise BrokenProcessPool(self._broken)
763767
_threads_wakeups[self._executor_manager_thread] = \
764768
self._executor_manager_thread_wakeup
765769

@@ -860,7 +864,10 @@ def shutdown(self, wait=True, *, cancel_futures=False):
860864
self._executor_manager_thread_wakeup.wakeup()
861865

862866
if self._executor_manager_thread is not None and wait:
863-
self._executor_manager_thread.join()
867+
try:
868+
self._executor_manager_thread.join()
869+
except RuntimeError:
870+
self._executor_manager_thread.join_executor_internals(broken=True)
864871
# To reduce the risk of opening too many files, remove references to
865872
# objects that use file descriptors.
866873
self._executor_manager_thread = None

‎Lib/multiprocessing/connection.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -491,6 +491,10 @@ def close(self):
491491
self._listener = None
492492
listener.close()
493493

494+
@property
495+
def closed(self):
496+
return self._listener is None
497+
494498
@property
495499
def address(self):
496500
return self._listener._address

‎Lib/multiprocessing/heap.py

Lines changed: 6 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -142,9 +142,6 @@ def __init__(self, size=mmap.PAGESIZE):
142142
self._allocated_blocks = defaultdict(set)
143143
self._arenas = []
144144

145-
# List of pending blocks to free - see comment in free() below
146-
self._pending_free_blocks = []
147-
148145
# Statistics
149146
self._n_mallocs = 0
150147
self._n_frees = 0
@@ -255,43 +252,16 @@ def _remove_allocated_block(self, block):
255252
# Arena is entirely free, discard it from this process
256253
self._discard_arena(arena)
257254

258-
def _free_pending_blocks(self):
259-
# Free all the blocks in the pending list - called with the lock held.
260-
while True:
261-
try:
262-
block = self._pending_free_blocks.pop()
263-
except IndexError:
264-
break
265-
self._add_free_block(block)
266-
self._remove_allocated_block(block)
267-
268255
def free(self, block):
269256
# free a block returned by malloc()
270-
# Since free() can be called asynchronously by the GC, it could happen
271-
# that it's called while self._lock is held: in that case,
272-
# self._lock.acquire() would deadlock (issue #12352). To avoid that, a
273-
# trylock is used instead, and if the lock can't be acquired
274-
# immediately, the block is added to a list of blocks to be freed
275-
# synchronously sometimes later from malloc() or free(), by calling
276-
# _free_pending_blocks() (appending and retrieving from a list is not
277-
# strictly thread-safe but under CPython it's atomic thanks to the GIL).
278257
if os.getpid() != self._lastpid:
279258
raise ValueError(
280259
"My pid ({0:n}) is not last pid {1:n}".format(
281260
os.getpid(),self._lastpid))
282-
if not self._lock.acquire(False):
283-
# can't acquire the lock right now, add the block to the list of
284-
# pending blocks to free
285-
self._pending_free_blocks.append(block)
286-
else:
287-
# we hold the lock
288-
try:
289-
self._n_frees += 1
290-
self._free_pending_blocks()
291-
self._add_free_block(block)
292-
self._remove_allocated_block(block)
293-
finally:
294-
self._lock.release()
261+
with self._lock:
262+
self._n_frees += 1
263+
self._add_free_block(block)
264+
self._remove_allocated_block(block)
295265

296266
def malloc(self, size):
297267
# return a block of right size (possibly rounded up)
@@ -303,8 +273,6 @@ def malloc(self, size):
303273
self.__init__() # reinitialize after fork
304274
with self._lock:
305275
self._n_mallocs += 1
306-
# allow pending blocks to be marked available
307-
self._free_pending_blocks()
308276
size = self._roundup(max(size, 1), self._alignment)
309277
(arena, start, stop) = self._malloc(size)
310278
real_stop = start + size
@@ -330,7 +298,8 @@ def __init__(self, size):
330298
raise OverflowError("Size {0:n} too large".format(size))
331299
block = BufferWrapper._heap.malloc(size)
332300
self._state = (block, size)
333-
util.Finalize(self, BufferWrapper._heap.free, args=(block,))
301+
util.Finalize(self, BufferWrapper._heap.free, args=(block,),
302+
reentrant=False)
334303

335304
def create_memoryview(self):
336305
(arena, start, stop), size = self._state

‎Lib/multiprocessing/managers.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -180,14 +180,15 @@ def serve_forever(self):
180180
except (KeyboardInterrupt, SystemExit):
181181
pass
182182
finally:
183+
self.listener.close()
183184
if sys.stdout != sys.__stdout__: # what about stderr?
184185
util.debug('resetting stdout, stderr')
185186
sys.stdout = sys.__stdout__
186187
sys.stderr = sys.__stderr__
187188
sys.exit(0)
188189

189190
def accepter(self):
190-
while True:
191+
while not self.listener.closed:
191192
try:
192193
c = self.listener.accept()
193194
except OSError:
@@ -575,7 +576,7 @@ def start(self, initializer=None, initargs=()):
575576
self, type(self)._finalize_manager,
576577
args=(self._process, self._address, self._authkey, self._state,
577578
self._Client, self._shutdown_timeout),
578-
exitpriority=0
579+
exitpriority=0, reentrant=False
579580
)
580581

581582
@classmethod
@@ -859,12 +860,11 @@ def _incref(self):
859860
self._idset.add(self._id)
860861

861862
state = self._manager and self._manager._state
862-
863863
self._close = util.Finalize(
864864
self, BaseProxy._decref,
865865
args=(self._token, self._authkey, state,
866866
self._tls, self._idset, self._Client),
867-
exitpriority=10
867+
exitpriority=10, reentrant=False
868868
)
869869

870870
@staticmethod

‎Lib/multiprocessing/pool.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ def __init__(self, processes=None, initializer=None, initargs=(),
219219
p.terminate()
220220
for p in self._pool:
221221
p.join()
222+
self._pool.clear()
222223
raise
223224

224225
sentinels = self._get_sentinels()
@@ -257,7 +258,7 @@ def __init__(self, processes=None, initializer=None, initargs=(),
257258
args=(self._taskqueue, self._inqueue, self._outqueue, self._pool,
258259
self._change_notifier, self._worker_handler, self._task_handler,
259260
self._result_handler, self._cache),
260-
exitpriority=15
261+
exitpriority=15, reentrant=False
261262
)
262263
self._state = RUN
263264

@@ -665,8 +666,8 @@ def join(self):
665666
self._worker_handler.join()
666667
self._task_handler.join()
667668
self._result_handler.join()
668-
for p in self._pool:
669-
p.join()
669+
while self._pool:
670+
self._pool.pop().join()
670671

671672
@staticmethod
672673
def _help_stuff_finish(inqueue, task_handler, size):

‎Lib/multiprocessing/queues.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,14 +201,14 @@ def _start_thread(self):
201201
self._jointhread = Finalize(
202202
self._thread, Queue._finalize_join,
203203
[weakref.ref(self._thread)],
204-
exitpriority=-5
204+
exitpriority=-5, reentrant=False
205205
)
206206

207207
# Send sentinel to the thread queue object when garbage collected
208208
self._close = Finalize(
209209
self, Queue._finalize_close,
210210
[self._buffer, self._notempty],
211-
exitpriority=10
211+
exitpriority=10, reentrant=False
212212
)
213213

214214
@staticmethod

‎Lib/multiprocessing/synchronize.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ def _after_fork(obj):
7979
from .resource_tracker import register
8080
register(self._semlock.name, "semaphore")
8181
util.Finalize(self, SemLock._cleanup, (self._semlock.name,),
82-
exitpriority=0)
82+
exitpriority=0, reentrant=False)
8383

8484
@staticmethod
8585
def _cleanup(name):

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy