Skip to content

Commit ff58c3c

Browse files
miss-islingtoncolesbury
miss-islington
and
colesbury
authored
[3.13] gh-123022: Fix crash with Py_Initialize in background thread (GH-123052) (#123114)
Check that the current default heap is initialized in `_mi_os_get_aligned_hint` and `mi_os_claim_huge_pages`. The mimalloc function `_mi_os_get_aligned_hint` assumes that there is an initialized default heap. This is true for our main thread, but not for background threads. The problematic code path is usually called during initialization (i.e., `Py_Initialize`), but it may also be called if the program allocates large amounts of memory in total. The crash only affected the free-threaded build. (cherry picked from commit d061ffe) Co-authored-by: Sam Gross <colesbury@gmail.com>
1 parent 6001dea commit ff58c3c

4 files changed

Lines changed: 40 additions & 5 deletions

File tree

‎Lib/test/test_embed.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Run the tests in Programs/_testembed.c (tests for the CPython embedding APIs)
22
from test import support
3-
from test.support import import_helper, os_helper, MS_WINDOWS
3+
from test.support import import_helper, os_helper, threading_helper, MS_WINDOWS
44
import unittest
55

66
from collections import namedtuple
@@ -1792,6 +1792,13 @@ def test_init_main_interpreter_settings(self):
17921792

17931793
self.assertEqual(out, expected)
17941794

1795+
@threading_helper.requires_working_threading()
1796+
def test_init_in_background_thread(self):
1797+
# gh-123022: Check that running Py_Initialize() in a background
1798+
# thread doesn't crash.
1799+
out, err = self.run_embedded_interpreter("test_init_in_background_thread")
1800+
self.assertEqual(err, "")
1801+
17951802

17961803
class SetConfigTests(unittest.TestCase):
17971804
def test_set_config(self):

‎Misc/NEWS.d/next/Core_and_Builtins/2024-08-15-19-28-43.gh-issue-123022.m3EF9E.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
Fix crash in free-threaded build when calling :c:func:`Py_Initialize` from
2+
a non-main thread.

‎Objects/mimalloc/os.c‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -115,8 +115,12 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size)
115115
if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize
116116
uintptr_t init = MI_HINT_BASE;
117117
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of aligned allocations unless in debug mode
118-
uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
119-
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
118+
mi_heap_t* heap = mi_prim_get_default_heap();
119+
// gh-123022: default heap may not be initialized in CPython in background threads
120+
if (mi_heap_is_initialized(heap)) {
121+
uintptr_t r = _mi_heap_random_next(heap);
122+
init = init + ((MI_SEGMENT_SIZE * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB
123+
}
120124
#endif
121125
uintptr_t expected = hint + size;
122126
mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init);
@@ -553,8 +557,12 @@ static uint8_t* mi_os_claim_huge_pages(size_t pages, size_t* total_size) {
553557
// Initialize the start address after the 32TiB area
554558
start = ((uintptr_t)32 << 40); // 32TiB virtual start address
555559
#if (MI_SECURE>0 || MI_DEBUG==0) // security: randomize start of huge pages unless in debug mode
556-
uintptr_t r = _mi_heap_random_next(mi_prim_get_default_heap());
557-
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
560+
mi_heap_t* heap = mi_prim_get_default_heap();
561+
// gh-123022: default heap may not be initialized in CPython in background threads
562+
if (mi_heap_is_initialized(heap)) {
563+
uintptr_t r = _mi_heap_random_next(heap);
564+
start = start + ((uintptr_t)MI_HUGE_OS_PAGE_SIZE * ((r>>17) & 0x0FFF)); // (randomly 12bits)*1GiB == between 0 to 4TiB
565+
}
558566
#endif
559567
}
560568
end = start + size;

‎Programs/_testembed.c‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include <Python.h>
99
#include "pycore_initconfig.h" // _PyConfig_InitCompatConfig()
1010
#include "pycore_runtime.h" // _PyRuntime
11+
#include "pycore_pythread.h" // PyThread_start_joinable_thread()
1112
#include "pycore_import.h" // _PyImport_FrozenBootstrap
1213
#include <inttypes.h>
1314
#include <stdio.h>
@@ -2022,6 +2023,22 @@ static int test_init_main_interpreter_settings(void)
20222023
return 0;
20232024
}
20242025

2026+
static void do_init(void *unused)
2027+
{
2028+
_testembed_Py_Initialize();
2029+
Py_Finalize();
2030+
}
2031+
2032+
static int test_init_in_background_thread(void)
2033+
{
2034+
PyThread_handle_t handle;
2035+
PyThread_ident_t ident;
2036+
if (PyThread_start_joinable_thread(&do_init, NULL, &ident, &handle) < 0) {
2037+
return -1;
2038+
}
2039+
return PyThread_join_thread(handle);
2040+
}
2041+
20252042

20262043
#ifndef MS_WINDOWS
20272044
#include "test_frozenmain.h" // M_test_frozenmain
@@ -2211,6 +2228,7 @@ static struct TestCase TestCases[] = {
22112228
{"test_get_argc_argv", test_get_argc_argv},
22122229
{"test_init_use_frozen_modules", test_init_use_frozen_modules},
22132230
{"test_init_main_interpreter_settings", test_init_main_interpreter_settings},
2231+
{"test_init_in_background_thread", test_init_in_background_thread},
22142232

22152233
// Audit
22162234
{"test_open_code_hook", test_open_code_hook},

0 commit comments

Comments
 (0)