gh-144319: Add huge pages support for pymalloc

pablogsal · Jan 29, 2026 · Jan 29, 2026 · Jan 29, 2026 · Jan 29, 2026 · 16ecc7e53bad60f9f6eeadbe7dac4104c3d1afe0
commit 16ecc7e53bad60f9f6eeadbe7dac4104c3d1afe0
@@ -677,7 +677,11 @@ The pymalloc allocator
 Python has a *pymalloc* allocator optimized for small objects (smaller or equal
 to 512 bytes) with a short lifetime. It uses memory mappings called "arenas"
 with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit
-platforms. It falls back to :c:func:`PyMem_RawMalloc` and
+platforms. When Python is configured with :option:`--with-pymalloc-hugepages`,
+the arena size on 64-bit platforms is increased to 2 MiB to match the huge page
+size, and arena allocation will attempt to use huge pages (``MAP_HUGETLB`` on
+Linux, ``MEM_LARGE_PAGES`` on Windows) with automatic fallback to regular pages.
+It falls back to :c:func:`PyMem_RawMalloc` and
 :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes.
 
 *pymalloc* is the :ref:`default allocator <default-memory-allocators>` of the

@@ -783,6 +783,18 @@ also be used to improve performance.
 
    See also :envvar:`PYTHONMALLOC` environment variable.
 
+.. option:: --with-pymalloc-hugepages
+
+   Enable huge page support for :ref:`pymalloc <pymalloc>` arenas (disabled by
+   default). When enabled, the arena size on 64-bit platforms is increased to
+   2 MiB and arena allocation uses ``MAP_HUGETLB`` (Linux) or
+   ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages.
+
+   The configure script checks that the platform supports ``MAP_HUGETLB``
+   and emits a warning if it is not available.
+
+   .. versionadded:: 3.15
+
 .. option:: --without-doc-strings
 
    Disable static documentation strings to reduce the memory footprint (enabled

@@ -1477,6 +1477,11 @@ Build changes
   modules that are missing or packaged separately.
   (Contributed by Stan Ulbrych and Petr Viktorin in :gh:`139707`.)
 
+* The new configure option :option:`--with-pymalloc-hugepages` enables huge
+  page support for :ref:`pymalloc <pymalloc>` arenas. When enabled, arena size
+  increases to 2 MiB and allocation uses ``MAP_HUGETLB`` (Linux) or
+  ``MEM_LARGE_PAGES`` (Windows) with automatic fallback to regular pages.
+
 * Annotating anonymous mmap usage is now supported if Linux kernel supports
   :manpage:`PR_SET_VMA_ANON_NAME <PR_SET_VMA(2const)>` (Linux 5.17 or newer).
   Annotations are visible in ``/proc/<pid>/maps`` if the kernel supports the feature

@@ -208,7 +208,11 @@ typedef unsigned int pymem_uint;  /* assuming >= 16 bits */
  * mappings to reduce heap fragmentation.
  */
 #ifdef USE_LARGE_ARENAS
-#define ARENA_BITS              20                    /* 1 MiB */
+#  ifdef PYMALLOC_USE_HUGEPAGES
+#    define ARENA_BITS            21                    /* 2 MiB */
+#  else
+#    define ARENA_BITS            20                    /* 1 MiB */
+#  endif
 #else
 #define ARENA_BITS              18                    /* 256 KiB */
 #endif
@@ -469,7 +473,7 @@ nfp free pools in usable_arenas.
 */
 
 /* How many arena_objects do we initially allocate?
- * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the
+ * 16 = can allocate 16 arenas = 16 * ARENA_SIZE before growing the
  * `arenas` vector.
  */
 #define INITIAL_ARENA_OBJECTS 16
@@ -512,14 +516,26 @@ struct _obmalloc_mgmt {
 
    memory address bit allocation for keys
 
-   64-bit pointers, IGNORE_BITS=0 and 2^20 arena size:
+   ARENA_BITS is configurable: 20 (1 MiB) by default on 64-bit, or
+   21 (2 MiB) when PYMALLOC_USE_HUGEPAGES is enabled.  All bit widths
+   below are derived from ARENA_BITS automatically.
+
+   64-bit pointers, IGNORE_BITS=0 and 2^20 arena size (default):
      15 -> MAP_TOP_BITS
      15 -> MAP_MID_BITS
      14 -> MAP_BOT_BITS
      20 -> ideal aligned arena
    ----
      64
 
+   64-bit pointers, IGNORE_BITS=0 and 2^21 arena size (hugepages):
+     15 -> MAP_TOP_BITS
+     15 -> MAP_MID_BITS
+     13 -> MAP_BOT_BITS
+     21 -> ideal aligned arena
+   ----
+     64
+
    64-bit pointers, IGNORE_BITS=16, and 2^20 arena size:
      16 -> IGNORE_BITS
      10 -> MAP_TOP_BITS

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-01-29-01-42-14.gh-issue-144319._7EtdB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-01-29-01-42-14.gh-issue-144319._7EtdB.rst
@@ -0,0 +1 @@
+Add huge pages support for the pymalloc allocator. Patch by Pablo Galindo
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
@@ -496,10 +496,30 @@ void *
 _PyMem_ArenaAlloc(void *Py_UNUSED(ctx), size_t size)
 {
 #ifdef MS_WINDOWS
+#  ifdef PYMALLOC_USE_HUGEPAGES
+    void *ptr = VirtualAlloc(NULL, size,
+                    MEM_COMMIT | MEM_RESERVE | MEM_LARGE_PAGES,
+                    PAGE_READWRITE);
+    if (ptr != NULL)
+        return ptr;
+    /* Fall back to regular pages */
+#  endif
     return VirtualAlloc(NULL, size,
                         MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
 #elif defined(ARENAS_USE_MMAP)
     void *ptr;
+#  ifdef PYMALLOC_USE_HUGEPAGES
+#    ifdef MAP_HUGETLB
+    ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,
+               MAP_PRIVATE|MAP_ANONYMOUS|MAP_HUGETLB, -1, 0);
+    if (ptr != MAP_FAILED) {
+        assert(ptr != NULL);
+        (void)_PyAnnotateMemoryMap(ptr, size, "cpython:pymalloc:hugepage");
+        return ptr;
+    }
+    /* Fall back to regular pages */
+#    endif
+#  endif
     ptr = mmap(NULL, size, PROT_READ|PROT_WRITE,
                MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
     if (ptr == MAP_FAILED)

@@ -5061,6 +5061,29 @@ then
 fi
 AC_MSG_RESULT([$with_pymalloc])
 
+AC_MSG_CHECKING([for --with-pymalloc-hugepages])
+AC_ARG_WITH(
+  [pymalloc-hugepages],
+  [AS_HELP_STRING([--with-pymalloc-hugepages],
+    [enable huge page support for pymalloc arenas (default is no)])])
+if test "$with_pymalloc_hugepages" = "yes"
+then
+    dnl configure only runs on Unix-like systems; Windows uses MEM_LARGE_PAGES
+    dnl via VirtualAlloc but does not use configure. Only check MAP_HUGETLB here.
+    AC_COMPILE_IFELSE(
+      [AC_LANG_PROGRAM([[
+#include <sys/mman.h>
+      ]], [[
+int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB;
+(void)flags;
+      ]])],
+      [AC_DEFINE([PYMALLOC_USE_HUGEPAGES], [1],
+         [Define to use huge pages for pymalloc arenas])],
+      [AC_MSG_WARN([--with-pymalloc-hugepages requested but MAP_HUGETLB not found])
+       with_pymalloc_hugepages=no])
+fi
+AC_MSG_RESULT([${with_pymalloc_hugepages:-no}])
+
 # Check for --with-c-locale-coercion
 AC_MSG_CHECKING([for --with-c-locale-coercion])
 AC_ARG_WITH(

diff --git a/pyconfig.h.in b/pyconfig.h.in
@@ -1739,6 +1739,9 @@
 /* Define as the preferred size in bits of long digits */
 #undef PYLONG_BITS_IN_DIGIT
 
+/* Define to use huge pages for pymalloc arenas */
+#undef PYMALLOC_USE_HUGEPAGES
+
 /* enabled builtin hash modules */
 #undef PY_BUILTIN_HASHLIB_HASHES
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Add huge pages support for the pymalloc allocator. Patch by Pablo Galindo