Skip to content

gh-64192: Make imap()/imap_unordered() in multiprocessing.pool actually lazy #136871

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
encukou merged 35 commits into from
Jul 29, 2026
Merged

gh-64192: Make imap()/imap_unordered() in multiprocessing.pool actually lazy #136871

Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
ccd0bbc
draft: impl lazy input consumption in mp.Pool.imap(_unordered)
Jul 20, 2025
002ef46
Use semaphore to synchronize threads
Jul 20, 2025
6e0bc58
Update buffersize behavior to match concurrent.futures.Executor behavior
Jul 21, 2025
62b2b6a
Release all `buffersize_lock` obj from the parent thread when terminate
Jul 21, 2025
0b6ba41
Add 2 basic `ThreadPool.imap()` tests w/ and w/o buffersize
Jul 21, 2025
aade15e
Fix accidental swap in imports
Jul 21, 2025
fb38a72
clear Pool._taskqueue_buffersize_semaphores safely
Jul 21, 2025
6ef488b
Slightly optimize Pool._taskqueue_buffersize_semaphores terminate
Jul 21, 2025
1716725
Rename `Pool.imap()` buffersize-related tests
Jul 21, 2025
9b43cd0
Fix typo in `IMapIterator.__init__()`
Jul 22, 2025
2d89341
Add tests for buffersize combinations with other kwargs
Jul 22, 2025
9ab2705
Remove if-branch in `_terminate_pool`
Jul 27, 2025
a955003
Add more edge-case tests for `imap` and `imap_unodered`
Jul 27, 2025
80efd6e
Split inf iterable test for `imap` and `imap_unordered`
Jul 27, 2025
83d6930
Add doc for `buffersize` argument of `imap` and `imap_unordered`
Jul 27, 2025
995ad8c
add *versionadded* for `imap_unordered`
Jul 28, 2025
3b6ad65
Remove ambiguity in `buffersize` description.
Jul 28, 2025
c941c16
Set *versionadded* as next in docs
Jul 28, 2025
d09e891
Add whatsnew entry
Jul 28, 2025
9c6d89d
Fix aggreed comments on code formatting/minor refactoring
Jul 28, 2025
4550a01
Remove `imap` and `imap_unordered` body code duplication
Jul 28, 2025
77bde4d
Merge branch 'main' into feature/add-buffersize-to-multiprocessing
obaltian Aug 31, 2025
aec39fc
Merge branch 'main' into feature/add-buffersize-to-multiprocessing
obaltian Sep 3, 2025
a469f2d
Merge remote-tracking branch 'origin/main' into feature/add-buffersiz…
obaltian Jul 17, 2026
bcda6e7
Remove extra import
obaltian Jul 17, 2026
1441eaa
Update Lib/multiprocessing/pool.py
obaltian Jul 18, 2026
6d61bd1
Update Doc/library/multiprocessing.rst
obaltian Jul 18, 2026
0e43d9c
Apply suggestion from @encukou
obaltian Jul 18, 2026
4ed84bc
Make `buffersize` argument kwarg-only
Jul 18, 2026
f7aeb2e
Use set for `._taskqueue_buffersize_semaphores`
Jul 18, 2026
78b1f58
Make `buffersize` arg check inline
Jul 18, 2026
49b03d9
Simplify `.imap()` and `.imap_undered()` tests
Jul 18, 2026
455b748
Fix iterator instantiation in in `.imap()`
Jul 18, 2026
56eaf3f
Mark kw-only arg in `imap`/`imap_unordered` in doc
Jul 18, 2026
ea540bf
Merge branch 'main' into feature/add-buffersize-to-multiprocessing
obaltian Jul 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Split inf iterable test for imap and imap_unordered
  • Loading branch information
Oleksandr Baltian @obaltian
Oleksandr Baltian authored and obaltian committed Aug 14, 2025
commit 80efd6e56ed45aed25211b15fca0ff591c32ac3c
43 changes: 26 additions & 17 deletions Lib/test/_test_multiprocessing.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3138,32 +3138,41 @@ def produce_args():
p.terminate()
p.join()

def test_imap_and_imap_unordered_buffersize_on_infinite_iterable(self):
def test_imap_and_imap_unordered_buffersize_on_empty_iterable(self):
for method_name in ("imap", "imap_unordered"):
with self.subTest(method=method_name):
method = getattr(self.pool, method_name)

res = method(str, [], buffersize=2)

self.assertIsNone(next(res, None))

def test_imap_buffersize_on_infinite_iterable(self):
if self.TYPE != "threads":
self.skipTest("test not appropriate for {}".format(self.TYPE))

for method_name in ("imap", "imap_unordered"):
with self.subTest(method=method_name):
p = self.Pool(4)
method = getattr(p, method_name)
p = self.Pool(4)
res = p.imap(str, itertools.count(), buffersize=2)

res = method(str, itertools.count(), buffersize=2)
self.assertEqual(next(res, None), "0")
self.assertEqual(next(res, None), "1")
self.assertEqual(next(res, None), "2")

self.assertEqual(next(res, None), "0")
self.assertEqual(next(res, None), "1")
self.assertEqual(next(res, None), "2")
p.terminate()
p.join()

p.terminate()
p.join()
def test_imap_unordered_buffersize_on_infinite_iterable(self):
if self.TYPE != "threads":
self.skipTest("test not appropriate for {}".format(self.TYPE))

def test_imap_and_imap_unordered_buffersize_on_empty_iterable(self):
for method_name in ("imap", "imap_unordered"):
with self.subTest(method=method_name):
method = getattr(self.pool, method_name)
p = self.Pool(4)
res = p.imap(str, itertools.count(), buffersize=2)
first_three_results = sorted(next(res, None) for _ in range(3))

res = method(str, [], buffersize=2)
self.assertEqual(first_three_results, ["0", "1", "2"])
Comment thread
encukou marked this conversation as resolved.
Outdated

self.assertIsNone(next(res, None))
p.terminate()
p.join()

def test_make_pool(self):
expected_error = (RemoteError if self.TYPE == 'manager'
Expand Down