Skip to content

gh-120161: Clean Up the Managed Static Types State #120663

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ericsnowcurrently wants to merge 8 commits into from
Closed

gh-120161: Clean Up the Managed Static Types State #120663

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
e3057ce
Clean up managed_static_type_state_init() a little.
ericsnowcurrently Jun 4, 2024
049934b
Track managed static types globally, with a shared index pool.
ericsnowcurrently Jun 4, 2024
1503e29
Unify the two per-interpreter structs.
ericsnowcurrently Jun 4, 2024
c820fdc
Add some asserts.
ericsnowcurrently Jun 13, 2024
de98b0f
wip - Do not reset a static type's index.
ericsnowcurrently Jun 6, 2024
bbfafd8
Small cleanup.
ericsnowcurrently Jun 17, 2024
2a30616
Drop an ifdef.
ericsnowcurrently Jun 17, 2024
6a20de3
Silence some compiler warnings.
ericsnowcurrently Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Unify the two per-interpreter structs.
  • Loading branch information
@ericsnowcurrently
ericsnowcurrently committed Jun 17, 2024
commit 1503e29dc87834403b78390e4f40c6755dfef879
45 changes: 22 additions & 23 deletions Include/internal/pycore_typeobject.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,24 +61,14 @@ struct type_cache {
struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP];
};

typedef struct {
PyTypeObject *type;
int readying;
int ready;
// XXX tp_dict can probably be statically allocated,
// instead of dynamically and stored on the interpreter.
PyObject *tp_dict;
PyObject *tp_subclasses;
/* We never clean up weakrefs for static builtin types since
they will effectively never get triggered. However, there
are also some diagnostic uses for the list of weakrefs,
so we still keep it. */
PyObject *tp_weaklist;
} managed_static_type_state;

typedef struct managed_static_type_state managed_static_type_state;

#define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */

struct types_state {
PyMutex mutex;

/* Used to set PyTypeObject.tp_version_tag.
It starts at _Py_MAX_GLOBAL_TYPE_VERSION_TAG + 1,
where all those lower numbers are used for core static types. */
Expand Down Expand Up @@ -123,15 +113,24 @@ struct types_state {
builtin type. Once initialization is over for a subinterpreter,
the value will be the same as for all other interpreters. */
struct {
size_t num_initialized;
managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES];
} builtins;
/* We apply a similar strategy for managed extension modules. */
struct {
size_t num_initialized;
managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES];
} for_extensions;
PyMutex mutex;
size_t num_builtins;
size_t num_types;
struct managed_static_type_state {
//int initialized;
PyTypeObject *type;
int readying;
int ready;
// XXX tp_dict can probably be statically allocated,
// instead of dynamically and stored on the interpreter.
PyObject *tp_dict;
PyObject *tp_subclasses;
/* We never clean up weakrefs for static builtin types since
they will effectively never get triggered. However, there
are also some diagnostic uses for the list of weakrefs,
so we still keep it. */
PyObject *tp_weaklist;
} types[_Py_MAX_MANAGED_STATIC_TYPES];
} managed_static;

// Borrowed references to type objects whose
// tp_version_tag % TYPE_VERSION_CACHE_SIZE
Expand Down
101 changes: 48 additions & 53 deletions Objects/typeobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,9 @@ managed_static_type_index_init(PyInterpreterState *interp,
assert(_Py_IsMainInterpreter(interp));
assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
assert(index == _PyRuntime.types.managed_static.num_builtins);
assert(index == interp->types.builtins.num_initialized);
assert(index == interp->types.managed_static.num_builtins);
assert(index == _PyRuntime.types.managed_static.num_types);
assert(index == interp->types.managed_static.num_types);
}
else {
PyMutex_Lock(&_PyRuntime.types.managed_static.mutex);
Expand All @@ -170,13 +172,18 @@ managed_static_type_index_init(PyInterpreterState *interp,
_PyRuntime.types.managed_static.next_index = index + 1;
}
PyMutex_Unlock(&_PyRuntime.types.managed_static.mutex);
assert(index >= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
assert(index < _Py_MAX_MANAGED_STATIC_TYPES);

PyMutex_Lock(&interp->types.mutex);
assert((index - _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES)
== interp->types.for_extensions.num_initialized);
PyMutex_Unlock(&interp->types.mutex);
assert(index >= _PyRuntime.types.managed_static.num_builtins);
assert(index >= interp->types.managed_static.num_builtins);
assert(index == (_PyRuntime.types.managed_static.num_types +
_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES -
_PyRuntime.types.managed_static.num_builtins));
}
assert(_PyRuntime.types.managed_static.num_builtins ==
interp->types.managed_static.num_builtins);
assert(_PyRuntime.types.managed_static.num_types ==
interp->types.managed_static.num_types);

/* We store a 1-based index so 0 can mean "not initialized". */
self->tp_subclasses = (PyObject *)(index + 1);
Expand All @@ -188,17 +195,17 @@ managed_static_type_index_sync(PyInterpreterState *interp,
PyTypeObject *self, int isbuiltin)
{
size_t index = managed_static_type_index_get(self);
assert(index >= interp->types.managed_static.num_types);
assert(index < _Py_MAX_MANAGED_STATIC_TYPES);
if (isbuiltin) {
assert(index == interp->types.builtins.num_initialized);
assert(index == interp->types.managed_static.num_builtins);
assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
}
else {
// The following might not hold if there are more than one
// extension modules that have static types.
assert((index - _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES)
== interp->types.for_extensions.num_initialized);
assert(index >= interp->types.managed_static.num_builtins);
assert(index >= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
assert(index < _Py_MAX_MANAGED_STATIC_TYPES);
}
return index;
}
Expand All @@ -217,39 +224,28 @@ static_ext_type_lookup(PyInterpreterState *interp, size_t index,
assert(index < _Py_MAX_MANAGED_STATIC_TYPES);
assert(index >= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
assert(!_PyRuntime.types.managed_static.types[index].isbuiltin);
#ifndef NDEBUG
PyTypeObject *cached_type =
_PyRuntime.types.managed_static.types[index].type;
#endif

PyTypeObject *type = _PyRuntime.types.managed_static.types[index].type;
int64_t interp_count =
_PyRuntime.types.managed_static.types[index].interp_count;
assert((interp_count == 0) ==
(_PyRuntime.types.managed_static.types[index].type == NULL));
*p_interp_count = interp_count;

index -= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES;
PyTypeObject *type = interp->types.for_extensions.initialized[index].type;
if (type == NULL) {
assert(interp_count == 0);
*p_interp_count = interp_count;
return NULL;
}
assert(type == cached_type);
assert(interp_count > 0);
assert(interp->types.managed_static.types[index].type == type);
assert(managed_static_type_index_is_set(type));

*p_interp_count = interp_count;
return type;
}

static managed_static_type_state *
managed_static_type_state_get(PyInterpreterState *interp, PyTypeObject *self)
{
// It's probably a builtin type.
size_t index = managed_static_type_index_get(self);
if (index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES) {
return &(interp->types.builtins.initialized[index]);
}
else {
index -= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES;
return &(interp->types.for_extensions.initialized[index]);
}
return &(interp->types.managed_static.types[index]);
}

/* For static types we store some state in an array on each interpreter. */
Expand All @@ -271,7 +267,6 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self,
if (initial) {
index = managed_static_type_index_init(interp, self, isbuiltin);
assert(_PyRuntime.types.managed_static.types[index].interp_count == 0);

assert(_PyRuntime.types.managed_static.types[index].type == NULL);
_PyRuntime.types.managed_static.types[index].type = self;
_PyRuntime.types.managed_static.types[index].isbuiltin = isbuiltin;
Expand All @@ -284,10 +279,8 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self,
(void)_Py_atomic_add_int64(
&_PyRuntime.types.managed_static.types[index].interp_count, 1);

managed_static_type_state *state = isbuiltin
? &(interp->types.builtins.initialized[index])
: &(interp->types.for_extensions.initialized[
index - _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES]);
managed_static_type_state *state =
&(interp->types.managed_static.types[index]);

/* It should only be called once for each builtin type per interpreter. */
assert(state->type == NULL);
Expand All @@ -302,7 +295,8 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self,
_PyRuntime.types.managed_static.num_types += 1;
_PyRuntime.types.managed_static.num_builtins += 1;
}
interp->types.builtins.num_initialized += 1;
interp->types.managed_static.num_types += 1;
interp->types.managed_static.num_builtins += 1;
}
else {
if (initial) {
Expand All @@ -311,7 +305,7 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self,
PyMutex_Unlock(&_PyRuntime.types.managed_static.mutex);
}
PyMutex_Lock(&interp->types.mutex);
interp->types.for_extensions.num_initialized += 1;
interp->types.managed_static.num_types += 1;
PyMutex_Unlock(&interp->types.mutex);
}
}
Expand All @@ -323,7 +317,6 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self,
int isbuiltin, int final)
{
size_t index = managed_static_type_index_get(self);
managed_static_type_state *state;
if (isbuiltin) {
assert(index < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
assert(!final
Expand All @@ -333,7 +326,6 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self,
|| _PyRuntime.types.managed_static.next_index <=
_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
assert(_PyRuntime.types.managed_static.types[index].isbuiltin);
state = &(interp->types.builtins.initialized[index]);
}
else {
assert(index >= _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
Expand All @@ -342,9 +334,9 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self,
assert(_PyRuntime.types.managed_static.next_index >
_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES);
assert(!_PyRuntime.types.managed_static.types[index].isbuiltin);
state = &(interp->types.for_extensions.initialized[
index - _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES]);
}
managed_static_type_state *state =
&(interp->types.managed_static.types[index]);
assert(state != NULL);

assert(_PyRuntime.types.managed_static.types[index].type == state->type);
Expand All @@ -353,13 +345,9 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self,
&_PyRuntime.types.managed_static.types[index].interp_count, -1);

/* Update the various counts. */
interp->types.managed_static.num_types -= 1;
if (isbuiltin) {
interp->types.builtins.num_initialized -= 1;
}
else {
PyMutex_Lock(&interp->types.mutex);
interp->types.for_extensions.num_initialized -= 1;
PyMutex_Unlock(&interp->types.mutex);
interp->types.managed_static.num_builtins -= 1;
}
if (final) {
if (isbuiltin) {
Expand Down Expand Up @@ -951,14 +939,18 @@ _PyTypes_Fini(PyInterpreterState *interp)
struct type_cache *cache = &interp->types.type_cache;
type_cache_clear(cache, NULL);

assert(interp->types.managed_static.num_types == 0);
// All the managed static types should have been finalized already.
assert(interp->types.for_extensions.num_initialized == 0);
for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) {
assert(interp->types.for_extensions.initialized[i].type == NULL);
for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_TYPES; i++) {
assert(interp->types.managed_static.types[i].type == NULL);
}
assert(interp->types.builtins.num_initialized == 0);
for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) {
assert(interp->types.builtins.initialized[i].type == NULL);
if (_Py_IsMainInterpreter(interp)) {
assert(_PyRuntime.types.managed_static.num_types == 0);
// All the managed static types should have been finalized already.
for (size_t i = 0; i < _Py_MAX_MANAGED_STATIC_TYPES; i++) {
assert(_PyRuntime.types.managed_static.types[i].type == NULL);
assert(_PyRuntime.types.managed_static.types[i].interp_count == 0);
}
}
}

Expand Down Expand Up @@ -5954,7 +5946,10 @@ _PyTypes_FiniExtTypes(PyInterpreterState *interp)
for (size_t i = _Py_MAX_MANAGED_STATIC_TYPES;
i > _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i--)
{
if (interp->types.for_extensions.num_initialized == 0) {
/* There may be gaps. */
if (interp->types.managed_static.num_types ==
interp->types.managed_static.num_builtins)
{
break;
}
int64_t count = 0;
Expand Down