Skip to content

gh-155742: Check singletons consistency at Python exit #157572

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 12 commits into from
Sep 17, 2026
Merged

gh-155742: Check singletons consistency at Python exit #157572

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
cd83f8d
gh-155742: Check singletons consistencty at Python exit
vstinner Sep 15, 2026
121f23a
Fix compiler warnings
vstinner Sep 15, 2026
03d611a
Use loops in generated finalizer code
vstinner Sep 16, 2026
03f6eed
Address Petr's review
vstinner Sep 16, 2026
d195ee0
Always call _PyBytes_CheckOverflow()
vstinner Sep 16, 2026
17902a6
Apply suggestion from @vstinner
vstinner Sep 16, 2026
fd6eccb
Share more code in _PyStaticObject_CheckUnicode (#17)
encukou Sep 17, 2026
05cea1f
Add pycore_global_objects_fini.h internal header
vstinner Sep 17, 2026
bfa5de5
Update make regen-global-objects
vstinner Sep 17, 2026
809e911
Update test_capi
vstinner Sep 17, 2026
2cf826f
Merge branch 'main' into global_obj_fini_check
vstinner Sep 17, 2026
a155615
Move _PyHamt_BitmapNode_Type include
vstinner Sep 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3,415 changes: 983 additions & 2,432 deletions Include/internal/pycore_global_objects_fini_generated.h
View file
Open in desktop

Large diffs are not rendered by default.

46 changes: 46 additions & 0 deletions Lib/test/test_capi/test_misc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3094,5 +3094,51 @@ def test_ceval_decref(self):
self.assertEqual(lines.count("DESTROY list"), 2)


@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
class TestCheckSingleton(unittest.TestCase):
# Test that _PyStaticObjects_CheckAll() detect memory corruptions in
Comment thread
vstinner marked this conversation as resolved.
Resolved
# singleton objects at Python exit.

def check(self, code):
code = f"""if 1:
import _testcapi
from test import support
support.SuppressCrashReport().__enter__()
{code}
"""
proc = assert_python_failure("-c", code)
return proc.err

def test_corrupt_bytes(self):
stderr = self.check("_testcapi.corrupt_bytes(b'a', b'#')")

self.assertIn((b'_PyStaticObject_CheckBytesSingleton: '
b'Assertion "str[0] == ch" failed'), stderr)
self.assertIn(b"object repr : b'#'", stderr)

def test_corrupt_unicode(self):
stderr = self.check("_testcapi.corrupt_unicode('a', '#')")

self.assertIn((b'_PyStaticObject_CheckUnicodeCharSingleton: '
b'Assertion "data[0] == ch" failed'), stderr)
self.assertIn(b"object repr : '#'", stderr)

def test_corrupt_bool(self):
stderr = self.check("_testcapi.corrupt_long(True, 0)")

self.assertIn((b'_PyStaticObject_CheckLongSingleton: '
b'Assertion "compact == value" failed'),
stderr)
self.assertIn(b"object repr : True", stderr)

def test_corrupt_long(self):
stderr = self.check("_testcapi.corrupt_long(5, 42)")

self.assertIn((b'_PyStaticObject_CheckLongSingleton: '
b'Assertion "compact == value" failed'),
stderr)
self.assertIn(b"object repr : 42", stderr)


if __name__ == "__main__":
unittest.main()
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core_and_Builtins/2026-09-15-17-04-54.gh-issue-155742.2DUW_9.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
In debug mode at Python exit, check if immutable singleton objects have been
modified by mistake to detect bugs in C extensions. Patch by Victor Stinner.
16 changes: 16 additions & 0 deletions Modules/_testcapi/bytes.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,21 @@ bytearray_overflow(PyObject *Py_UNUSED(module), PyObject *args)
}


// Write into an immutable bytes object to test _PyStaticObjects_CheckAll()
static PyObject *
corrupt_bytes(PyObject *Py_UNUSED(module), PyObject *args)
{
char *bytes, *override;
Py_ssize_t size;
if (!PyArg_ParseTuple(args, "yy#", &bytes, &override, &size)) {
return NULL;
}

memcpy(bytes, override, size);
Py_RETURN_NONE;
}


static PyMethodDef test_methods[] = {
{"bytes_resize", bytes_resize, METH_VARARGS},
{"bytes_join", bytes_join, METH_VARARGS},
Expand All @@ -576,6 +591,7 @@ static PyMethodDef test_methods[] = {
{"test_byteswriter_ptr", test_byteswriter_ptr, METH_NOARGS},
{"bytes_overflow", bytes_overflow, METH_VARARGS},
{"bytearray_overflow", bytearray_overflow, METH_VARARGS},
{"corrupt_bytes", corrupt_bytes, METH_VARARGS},
{NULL},
};

Expand Down
16 changes: 16 additions & 0 deletions Modules/_testcapi/long.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,21 @@ get_pylong_layout(PyObject *module, PyObject *Py_UNUSED(args))
}


// Write into an immutable int object to test _PyStaticObjects_CheckAll()
static PyObject *
corrupt_long(PyObject *Py_UNUSED(module), PyObject *args)
{
PyObject *obj;
int value;
if (!PyArg_ParseTuple(args, "Oi", &obj, &value)) {
return NULL;
}

((PyLongObject*)obj)->long_value.ob_digit[0] = value;
Py_RETURN_NONE;
}


static PyMethodDef test_methods[] = {
_TESTCAPI_CALL_LONG_COMPACT_API_METHODDEF
{"pylong_fromunicodeobject", pylong_fromunicodeobject, METH_VARARGS},
Expand All @@ -295,6 +310,7 @@ static PyMethodDef test_methods[] = {
{"pylong_ispositive", pylong_ispositive, METH_O},
{"pylong_isnegative", pylong_isnegative, METH_O},
{"pylong_iszero", pylong_iszero, METH_O},
{"corrupt_long", corrupt_long, METH_VARARGS},
{NULL},
};

Expand Down
20 changes: 20 additions & 0 deletions Modules/_testcapi/unicode.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,25 @@ static PyType_Spec Writer_spec = {
};


// Write into an immutable str object to test _PyStaticObjects_CheckAll()
static PyObject *
corrupt_unicode(PyObject *Py_UNUSED(module), PyObject *args)
{
PyObject *obj, *override;
if (!PyArg_ParseTuple(args, "OO", &obj, &override)) {
return NULL;
}
assert(PyUnicode_KIND(obj) == PyUnicode_1BYTE_KIND);
assert(PyUnicode_KIND(override) == PyUnicode_1BYTE_KIND);

Py_UCS1 *dst = PyUnicode_1BYTE_DATA(obj);
Py_UCS1 *src = PyUnicode_1BYTE_DATA(override);
Py_ssize_t size = PyUnicode_GET_LENGTH(override);
memcpy(dst, src, size);
Py_RETURN_NONE;
}


static PyMethodDef TestMethods[] = {
{"unicode_new", unicode_new, METH_VARARGS},
{"unicode_fill", unicode_fill, METH_VARARGS},
Expand All @@ -572,6 +591,7 @@ static PyMethodDef TestMethods[] = {
{"unicode_asutf8", unicode_asutf8, METH_VARARGS},
{"unicode_copycharacters", unicode_copycharacters, METH_VARARGS},
{"unicode_GET_CACHED_HASH", unicode_GET_CACHED_HASH, METH_O},
{"corrupt_unicode", corrupt_unicode, METH_VARARGS},
{NULL},
};

Expand Down
4 changes: 2 additions & 2 deletions Python/pylifecycle.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
#include "pycore_fileutils.h" // _Py_ResetForceASCII()
#include "pycore_floatobject.h" // _PyFloat_InitTypes()
#include "pycore_freelist.h" // _PyObject_ClearFreeLists()
#include "pycore_global_objects_fini_generated.h" // _PyStaticObjects_CheckRefcnt()
#include "pycore_global_objects_fini_generated.h" // _PyStaticObjects_CheckAll()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_interpolation.h" // _PyInterpolation_InitTypes()
#include "pycore_long.h" // _PyLong_InitTypes()
Expand Down Expand Up @@ -2104,7 +2104,7 @@ finalize_interp_types(PyInterpreterState *interp)
#endif

#ifdef Py_DEBUG
_PyStaticObjects_CheckRefcnt(interp);
_PyStaticObjects_CheckAll(interp);
#endif
}

Expand Down
97 changes: 81 additions & 16 deletions Tools/build/generate_global_objects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,11 +137,34 @@

NON_GENERATED_IMMORTAL_OBJECTS = [
# The generated ones come from generate_runtime_init().
'(PyObject *)&_Py_SINGLETON(bytes_empty)',
'(PyObject *)&_Py_SINGLETON(tuple_empty)',
'(PyObject *)&_Py_SINGLETON(hamt_bitmap_node_empty)',
'(PyObject *)&_Py_INTERP_SINGLETON(interp, hamt_empty)',
'(PyObject *)&_Py_SINGLETON(context_token_missing)',

# b'' (empty bytes string)
('_PyStaticObject_CheckBytesSingleton({}, 0, 0);',
'(PyObject *)&_Py_SINGLETON(bytes_empty)'),
# () (empty tuple)
('_PyStaticObject_CheckSingleton({}, &PyTuple_Type);',
'(PyObject *)&_Py_SINGLETON(tuple_empty)'),
('_PyStaticObject_CheckSingleton({}, &_PyHamt_BitmapNode_Type);',
'(PyObject *)&_Py_SINGLETON(hamt_bitmap_node_empty)'),
('_PyStaticObject_CheckSingleton({}, &_PyHamt_Type);',
'(PyObject *)&_Py_INTERP_SINGLETON(interp, hamt_empty)'),
('_PyStaticObject_CheckSingleton({}, &_PyContextTokenMissing_Type);',
'(PyObject *)&_Py_SINGLETON(context_token_missing)'),
# False
('_PyStaticObject_CheckLongSingleton({}, 0, 1);',
'(PyObject *)&_Py_FalseStruct '),
# True
('_PyStaticObject_CheckLongSingleton({}, 1, 1);',
'(PyObject *)&_Py_TrueStruct'),
# None
('_PyStaticObject_CheckSingleton({}, &_PyNone_Type);',
'(PyObject *)&_Py_NoneStruct'),
# Ellipsis (...)
('_PyStaticObject_CheckSingleton({}, &PyEllipsis_Type);',
'(PyObject *)&_Py_EllipsisObject'),
# Py_NotImplemented
('_PyStaticObject_CheckSingleton({}, &_PyNotImplemented_Type);',
'(PyObject *)&_Py_NotImplementedStruct'),
]


Expand Down Expand Up @@ -299,28 +322,45 @@ def generate_runtime_init(identifiers, strings):
with printer.block('#define _Py_small_ints_INIT', continuation=True):
for i in range(-nsmallnegints, nsmallposints):
printer.write(f'_PyLong_DIGIT_INIT({i}),')
immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + {i}]')
immortal_objects.append((
None,
f'(PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + {i}]',
))
printer.write('')
with printer.block('#define _Py_bytes_characters_INIT', continuation=True):
for i in range(256):
printer.write(f'_PyBytes_CHAR_INIT({i}),')
immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(bytes_characters)[{i}]')
immortal_objects.append((
None,
f'(PyObject *)&_Py_SINGLETON(bytes_characters)[{i}]',
))
printer.write('')
with printer.block('#define _Py_str_literals_INIT', continuation=True):
for literal, name in sorted(strings.items(), key=lambda x: x[1]):
printer.write(f'INIT_STR({name}, "{literal}"),')
immortal_objects.append(f'(PyObject *)&_Py_STR({name})')
escaped = literal.replace("{", "{{").replace("}", "}}")
immortal_objects.append((
f'_PyStaticObject_CheckUnicodeSingleton({{}}, "{escaped}", {len(literal)});',
f'(PyObject *)&_Py_STR({name})',
))
printer.write('')
with printer.block('#define _Py_str_identifiers_INIT', continuation=True):
for name in sorted(identifiers):
assert name.isidentifier(), name
printer.write(f'INIT_ID({name}),')
immortal_objects.append(f'(PyObject *)&_Py_ID({name})')
assert all(ch not in "{}" for ch in name)
immortal_objects.append((
f'_PyStaticObject_CheckUnicodeSingleton({{}}, "{name}", {len(name)});',
f'(PyObject *)&_Py_ID({name})',
))
printer.write('')
with printer.block('#define _Py_str_ascii_INIT', continuation=True):
for i in range(128):
printer.write(f'_PyASCIIObject_INIT("\\x{i:02x}"),')
immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(strings).ascii[{i}]')
immortal_objects.append((
None,
f'(PyObject *)&_Py_SINGLETON(strings).ascii[{i}]',
))
printer.write('')
with printer.block('#define _Py_str_latin1_INIT', continuation=True):
for i in range(128, 256):
Expand All @@ -329,7 +369,10 @@ def generate_runtime_init(identifiers, strings):
utf8.append(f"\\x{c:02x}")
utf8.append('"')
printer.write(f'_PyUnicode_LATIN1_INIT("\\x{i:02x}", {"".join(utf8)}),')
immortal_objects.append(f'(PyObject *)&_Py_SINGLETON(strings).latin1[{i} - 128]')
immortal_objects.append((
None,
f'(PyObject *)&_Py_SINGLETON(strings).latin1[{i} - 128]',
))
printer.write(END)
printer.write(after)
return immortal_objects
Expand Down Expand Up @@ -373,6 +416,8 @@ def generate_static_strings_initializer(identifiers, strings):


def generate_global_object_finalizers(generated_immortal_objects):
nsmallnegints, nsmallposints = consts_getter.get_nsmallnegints_and_nsmallposints()

# Target the runtime initializer.
filename = os.path.join(INTERNAL, 'pycore_global_objects_fini_generated.h')

Expand All @@ -391,16 +436,36 @@ def generate_global_object_finalizers(generated_immortal_objects):
printer.write(before)
printer.write(START)
printer.write('#ifdef Py_DEBUG')
printer.write('// Check static objects consistency: detect if a C extension modified')
printer.write('// an immutable singleton by mistake.')
printer.write("static inline void")
with printer.block(
"_PyStaticObjects_CheckRefcnt(PyInterpreterState *interp)"):
"_PyStaticObjects_CheckAll(PyInterpreterState *interp)"):
printer.write('/* generated runtime-global */')
printer.write('// (see pycore_runtime_init_generated.h)')
for ref in generated_immortal_objects:
printer.write(f'_PyStaticObject_CheckRefcnt({ref});')
for fmt, ref in generated_immortal_objects:
if fmt is not None:
printer.write(fmt.format(ref))

with printer.block(f'for (int i={-nsmallnegints}; i < {nsmallposints}; i++)'):
obj = '(PyObject *)&_Py_SINGLETON(small_ints)[_PY_NSMALLNEGINTS + i]'
printer.write(f'_PyStaticObject_CheckLongSingleton({obj}, i, 0);')

with printer.block(f'for (int i=0; i <= 255; i++)'):
obj = '(PyObject *)&_Py_SINGLETON(bytes_characters)[i]'
printer.write(f'_PyStaticObject_CheckBytesSingleton({obj}, 1, i);')

with printer.block(f'for (int i=0; i <= 127; i++)'):
obj = '(PyObject *)&_Py_SINGLETON(strings).ascii[i]'
printer.write(f'_PyStaticObject_CheckUnicodeCharSingleton({obj}, i);')

with printer.block(f'for (int i=128; i <= 255; i++)'):
obj = '(PyObject *)&_Py_SINGLETON(strings).latin1[i - 128]'
printer.write(f'_PyStaticObject_CheckUnicodeCharSingleton({obj}, i);')

printer.write('/* non-generated */')
for ref in NON_GENERATED_IMMORTAL_OBJECTS:
printer.write(f'_PyStaticObject_CheckRefcnt({ref});')
for fmt, ref in NON_GENERATED_IMMORTAL_OBJECTS:
printer.write(fmt.format(ref))
printer.write('#endif // Py_DEBUG')
printer.write(END)
printer.write(after)
Expand Down
Loading