Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion Doc/library/multiprocessing.rst
Original file line number Diff line number Diff line change
Expand Up @@ -160,11 +160,21 @@ to start a process. These *start methods* are
No unnecessary resources are inherited.

Available on POSIX platforms which support passing file descriptors over
Unix pipes such as Linux. The default on those.
Unix pipes such as Linux. The default on those, unless there is no
writable temporary directory (see :func:`tempfile.gettempdir`), which
the fork server requires; *spawn* is the default in that case.

.. versionchanged:: 3.14
This became the default start method on POSIX platforms.

.. versionchanged:: next
Comment thread
gpshead marked this conversation as resolved.
*spawn* is now the default start method when there is no writable
temporary directory, for example on a read-only file system.
Previously *forkserver* remained the default there and starting a
process failed with an :exc:`OSError`. The default is determined
when it is first needed rather than when :mod:`!multiprocessing`
is imported. See :gh:`155717`.

.. versionchanged:: 3.4
*spawn* added on all POSIX platforms, and *forkserver* added for
some POSIX platforms.
Expand Down
47 changes: 31 additions & 16 deletions Lib/multiprocessing/context.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import functools
import os
import sys
import threading

from . import process
from . import reduction
from . import util

__all__ = ()

Expand Down Expand Up @@ -240,10 +240,19 @@ def _after_fork():
class DefaultContext(BaseContext):
Process = Process

def __init__(self, context):
self._default_context = context
def __init__(self, context=None):
if context is not None:
self._default_context = context
self._actual_context = None

# The platform default is only determined when first needed, as doing
# so can be costly; importing multiprocessing must stay cheap.
# A cached_property rather than a property so that the attribute can
# still be assigned, deleted and patched like a plain one.
@functools.cached_property
def _default_context(self):
return _get_platform_default_context()

def get_context(self, method=None):
if method is None:
if self._actual_context is None:
Expand Down Expand Up @@ -331,18 +340,21 @@ def _check_available(self):
if reduction.HAVE_SEND_HANDLE:
_concrete_contexts['forkserver'] = ForkServerContext()

# bpo-33725: running arbitrary code after fork() is no longer reliable
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
# gh-84559: We changed everyones default to a thread safeish one in 3.14.
if (
reduction.HAVE_SEND_HANDLE
and sys.platform != 'darwin'
# gh-155717: forkserver requires to write temporary files
and util._has_writeable_tempdir()
):
_default_context = DefaultContext(_concrete_contexts['forkserver'])
else:
_default_context = DefaultContext(_concrete_contexts['spawn'])
def _get_platform_default_context():
# bpo-33725: running arbitrary code after fork() is no longer reliable
# on macOS since macOS 10.14 (Mojave). Use spawn by default instead.
# gh-84559: We changed everyones default to a thread safeish one in 3.14.
if reduction.HAVE_SEND_HANDLE and sys.platform != 'darwin':
# gh-155717: forkserver requires to write temporary files.
#
# Imported here rather than at the top level as util is costly
# to import and is not otherwise needed to import this module.
from . import util
if util._has_writeable_tempdir():
return _concrete_contexts['forkserver']
return _concrete_contexts['spawn']

_default_context = DefaultContext()

else: # Windows

Expand All @@ -365,7 +377,10 @@ class SpawnContext(BaseContext):
_concrete_contexts = {
'spawn': SpawnContext(),
}
_default_context = DefaultContext(_concrete_contexts['spawn'])
def _get_platform_default_context():
return _concrete_contexts['spawn']

_default_context = DefaultContext()

#
# Force the start method
Expand Down
17 changes: 12 additions & 5 deletions Lib/multiprocessing/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import os
import itertools
import sys
import tempfile
import weakref
import atexit
import threading # we want threading to install it's
Expand Down Expand Up @@ -155,7 +154,7 @@ def _remove_temp_dir(rmtree, tempdir):
current_process._config['tempdir'] = None


def _get_base_temp_dir():
def _get_base_temp_dir(tempfile):
Comment thread
gpshead marked this conversation as resolved.
"""Get a temporary directory where socket files will be created.

To prevent additional imports, pass a pre-imported 'tempfile' module.
Expand Down Expand Up @@ -216,8 +215,9 @@ def get_temp_dir():
# get name of a temp directory which will be automatically cleaned up
tempdir = process.current_process()._config.get('tempdir')
if tempdir is None:
import shutil
base_tempdir = _get_base_temp_dir()
# Deferred until first use to keep importing this module cheap.
Comment thread
gpshead marked this conversation as resolved.
import shutil, tempfile
base_tempdir = _get_base_temp_dir(tempfile)
tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir)
info('created temp directory %s', tempdir)
# keep a strong reference to shutil.rmtree(), since the finalizer
Expand All @@ -234,7 +234,14 @@ def _has_writeable_tempdir():
#
# See: https://github.com/python/cpython/issues/155717.

path = _get_base_temp_dir()
# Deferred until first use to keep importing this module cheap.
import tempfile

try:
path = _get_base_temp_dir(tempfile)
except OSError:
# No usable temporary directory at all.
return False
if path is None:
return False

Expand Down
56 changes: 56 additions & 0 deletions Lib/test/_test_multiprocessing.py
Original file line number Diff line number Diff line change
Expand Up @@ -6516,6 +6516,62 @@ def test_forkserver_requires_writeable_tempdir(self, mode):
_, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR)
self.assertEqual(out.decode().strip(), "spawn")

@unittest.skipIf(os.name == "nt", "requires POSIX")
@support.requires_non_root_user
def test_default_start_method_without_usable_tempdir(self):
# Neither importing multiprocessing nor determining the default
# start method may fail when tempfile finds no usable temporary
# directory at all, and the default start method must be 'spawn'.
#
# See https://github.com/python/cpython/issues/155717.

cmd = '''if 1:
import os, tempfile
# Make TMPDIR the only candidate, so that tempfile.gettempdir()
# raises FileNotFoundError instead of falling back to /tmp.
tempfile._candidate_tempdir_list = lambda: [os.environ["TMPDIR"]]
try:
tempfile.gettempdir()
except FileNotFoundError:
pass
else:
raise AssertionError("TMPDIR is usable")

import multiprocessing
if __name__ == "__main__":
print(multiprocessing.get_start_method(),
multiprocessing.get_all_start_methods()[0])
'''

with support.os_helper.temp_dir() as root:
TMPDIR = pathlib.Path(root, "TMPDIR")
TMPDIR.mkdir(mode=os.R_OK | os.X_OK)
_, out, err = script_helper.assert_python_ok('-c', cmd, TMPDIR=TMPDIR)
self.assertEqual(out.decode().split(), ["spawn", "spawn"])

def test_import_does_not_determine_default_start_method(self):
# Determining the default start method requires checking that the
# temporary directory is writeable. That must not happen, nor may
# the modules it needs be loaded, when importing multiprocessing.
import_helper.ensure_lazy_imports(
"multiprocessing", {"tempfile", "shutil", "subprocess"})

cmd = '''if 1:
import sys
import multiprocessing
print(multiprocessing.get_start_method(allow_none=True),
"tempfile" in sys.modules,
multiprocessing.get_start_method(),
multiprocessing.get_all_start_methods()[0])
'''
# -S: site customizations could import tempfile themselves.
_, out, err = script_helper.assert_python_ok('-S', '-c', cmd)
unset, probed, method, first = out.decode().split()
self.assertEqual(unset, "None")
self.assertEqual(probed, "False")
self.assertEqual(method, first)
self.assertIn(method, multiprocessing.get_all_start_methods())


@unittest.skipIf(sys.platform == "win32",
"test semantics don't make sense on Windows")
Expand Down
Loading