Skip to content

GH-95909: Make _PyArg_Parser initialization thread safe #95958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
miss-islington merged 6 commits into from
Aug 16, 2022
Merged

GH-95909: Make _PyArg_Parser initialization thread safe #95958

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ab5eecc
add locking
kumaraditya303 Aug 13, 2022
242efec
fix type
kumaraditya303 Aug 15, 2022
36243aa
fix lock
kumaraditya303 Aug 15, 2022
58e6b55
remove atomics
kumaraditya303 Aug 16, 2022
4c6ffcf
refactor
kumaraditya303 Aug 16, 2022
47da017
fix style
kumaraditya303 Aug 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
add locking
  • Loading branch information
@kumaraditya303
kumaraditya303 authored Aug 15, 2022
commit ab5eecc3d59d5743437529780e2800a06b7cff6f
4 changes: 3 additions & 1 deletion Include/cpython/modsupport.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@ PyAPI_FUNC(PyObject **) _Py_VaBuildStack(
Py_ssize_t *p_nargs);

typedef struct _PyArg_Parser {
int initialized;
const char *format;
const char * const *keywords;
const char *fname;
Expand All @@ -60,6 +59,9 @@ typedef struct _PyArg_Parser {
int min; /* minimal number of arguments */
int max; /* maximal number of positional arguments */
PyObject *kwtuple; /* tuple of keyword parameter names */
struct {
int _value;
} initialized;
struct _PyArg_Parser *next;
} _PyArg_Parser;

Expand Down
4 changes: 4 additions & 0 deletions Include/internal/pycore_runtime.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ extern "C" {
#include "pycore_interp.h" // PyInterpreterState
#include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids

struct _getargs_runtime_state {
PyThread_type_lock mutex;
};

/* ceval state */

Expand Down Expand Up @@ -114,6 +117,7 @@ typedef struct pyruntimestate {

struct _ceval_runtime_state ceval;
struct _gilstate_runtime_state gilstate;
struct _getargs_runtime_state getargs;

PyPreConfig preconfig;

Expand Down
19 changes: 15 additions & 4 deletions Python/getargs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1979,10 +1979,18 @@ parser_init(struct _PyArg_Parser *parser)
const char * const *keywords = parser->keywords;
assert(keywords != NULL);

if (parser->initialized) {
if (_Py_atomic_load_relaxed(&parser->initialized)) {
assert(parser->kwtuple != NULL);
return 1;
}
PyThread_acquire_lock(_PyRuntime.getargs.mutex, WAIT_LOCK);
Comment thread
kumaraditya303 marked this conversation as resolved.
Outdated
// Check again if another thread initialized the parser
// while we were waiting for the lock.
if (_Py_atomic_load_relaxed(&parser->initialized)) {
assert(parser->kwtuple != NULL);
return 1;
}

assert(parser->pos == 0 &&
(parser->format == NULL || parser->fname == NULL) &&
parser->custom_msg == NULL &&
Expand All @@ -1991,6 +1999,7 @@ parser_init(struct _PyArg_Parser *parser)

int len, pos;
if (scan_keywords(keywords, &len, &pos) < 0) {
PyThread_release_lock(_PyRuntime.getargs.mutex);
return 0;
}

Expand All @@ -2000,6 +2009,7 @@ parser_init(struct _PyArg_Parser *parser)
assert(parser->fname == NULL);
if (parse_format(parser->format, len, pos,
&fname, &custommsg, &min, &max) < 0) {
PyThread_release_lock(_PyRuntime.getargs.mutex);
return 0;
}
}
Expand All @@ -2013,6 +2023,7 @@ parser_init(struct _PyArg_Parser *parser)
if (kwtuple == NULL) {
kwtuple = new_kwtuple(keywords, len, pos);
if (kwtuple == NULL) {
PyThread_release_lock(_PyRuntime.getargs.mutex);
return 0;
}
owned = 1;
Expand All @@ -2027,18 +2038,18 @@ parser_init(struct _PyArg_Parser *parser)
parser->min = min;
parser->max = max;
parser->kwtuple = kwtuple;
parser->initialized = owned ? 1 : -1;

assert(parser->next == NULL);
parser->next = static_arg_parsers;
static_arg_parsers = parser;
_Py_atomic_store_relaxed(&parser->initialized, owned ? 1 : -1);
PyThread_release_lock(_PyRuntime.getargs.mutex);
return 1;
}

static void
parser_clear(struct _PyArg_Parser *parser)
{
if (parser->initialized == 1) {
if (_Py_atomic_load_relaxed(&parser->initialized) == 1) {
Py_CLEAR(parser->kwtuple);
}
}
Expand Down
27 changes: 21 additions & 6 deletions Python/pystate.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ _Py_COMP_DIAG_POP

static int
alloc_for_runtime(PyThread_type_lock *plock1, PyThread_type_lock *plock2,
PyThread_type_lock *plock3)
PyThread_type_lock *plock3, PyThread_type_lock *plock4)
{
/* Force default allocator, since _PyRuntimeState_Fini() must
use the same allocator than this function. */
Expand All @@ -82,11 +82,20 @@ alloc_for_runtime(PyThread_type_lock *plock1, PyThread_type_lock *plock2,
return -1;
}

PyThread_type_lock lock4 = PyThread_allocate_lock();
if (lock4 == NULL) {
PyThread_free_lock(lock1);
PyThread_free_lock(lock2);
PyThread_free_lock(lock3);
return -1;
}

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

*plock1 = lock1;
*plock2 = lock2;
*plock3 = lock3;
*plock4 = lock4;
return 0;
}

Expand All @@ -97,7 +106,8 @@ init_runtime(_PyRuntimeState *runtime,
Py_ssize_t unicode_next_index,
PyThread_type_lock unicode_ids_mutex,
PyThread_type_lock interpreters_mutex,
PyThread_type_lock xidregistry_mutex)
PyThread_type_lock xidregistry_mutex,
PyThread_type_lock getargs_mutex)
{
if (runtime->_initialized) {
Py_FatalError("runtime already initialized");
Expand All @@ -119,6 +129,8 @@ init_runtime(_PyRuntimeState *runtime,

runtime->xidregistry.mutex = xidregistry_mutex;

runtime->getargs.mutex = getargs_mutex;

// Set it to the ID of the main thread of the main interpreter.
runtime->main_thread = PyThread_get_thread_ident();

Expand All @@ -141,8 +153,8 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
// is called multiple times.
Py_ssize_t unicode_next_index = runtime->unicode_ids.next_index;

PyThread_type_lock lock1, lock2, lock3;
if (alloc_for_runtime(&lock1, &lock2, &lock3) != 0) {
PyThread_type_lock lock1, lock2, lock3, lock4;
if (alloc_for_runtime(&lock1, &lock2, &lock3, &lock4) != 0) {
return _PyStatus_NO_MEMORY();
}

Expand All @@ -152,7 +164,7 @@ _PyRuntimeState_Init(_PyRuntimeState *runtime)
memcpy(runtime, &initial, sizeof(*runtime));
}
init_runtime(runtime, open_code_hook, open_code_userdata, audit_hook_head,
unicode_next_index, lock1, lock2, lock3);
unicode_next_index, lock1, lock2, lock3, lock4);

return _PyStatus_OK();
}
Expand All @@ -172,6 +184,7 @@ _PyRuntimeState_Fini(_PyRuntimeState *runtime)
FREE_LOCK(runtime->interpreters.mutex);
FREE_LOCK(runtime->xidregistry.mutex);
FREE_LOCK(runtime->unicode_ids.lock);
FREE_LOCK(runtime->getargs.mutex);

#undef FREE_LOCK
PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
Expand All @@ -194,6 +207,7 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)
int reinit_interp = _PyThread_at_fork_reinit(&runtime->interpreters.mutex);
int reinit_xidregistry = _PyThread_at_fork_reinit(&runtime->xidregistry.mutex);
int reinit_unicode_ids = _PyThread_at_fork_reinit(&runtime->unicode_ids.lock);
int reinit_getargs = _PyThread_at_fork_reinit(&runtime->getargs.mutex);

PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);

Expand All @@ -204,7 +218,8 @@ _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime)
if (reinit_interp < 0
|| reinit_main_id < 0
|| reinit_xidregistry < 0
|| reinit_unicode_ids < 0)
|| reinit_unicode_ids < 0
|| reinit_getargs < 0)
{
return _PyStatus_ERR("Failed to reinitialize runtime locks");

Expand Down