Skip to content

Commit 4b27f3a

Browse files
colesburycolesbury
committed
pythongh-117376: Make Py_DECREF a macro in ceval.c in free-threaded build
`Py_DECREF` and `PyStackRef_CLOSE` are now implemented as macros in the free-threaded build in ceval.c. There are two motivations; * MSVC has problems inlining functions in ceval.c in the PGO build. * We will want to mark escaping calls in order to spill the stack pointer in ceval.c and we will want to do this around `_Py_Dealloc` (or `_Py_MergeZeroLocalRefcount` or `_Py_DecRefShared`), not around the entire `Py_DECREF` or `PyStackRef_CLOSE` call.
1 parent db6f5e1 commit 4b27f3a

2 files changed

Lines changed: 55 additions & 27 deletions

File tree

‎Include/internal/pycore_stackref.h‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -191,18 +191,15 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj)
191191
} while (0)
192192

193193
#ifdef Py_GIL_DISABLED
194-
static inline void
195-
PyStackRef_CLOSE(_PyStackRef stackref)
196-
{
197-
if (PyStackRef_IsDeferred(stackref)) {
198-
// No assert for being immortal or deferred here.
199-
// The GC unsets deferred objects right before clearing.
200-
return;
201-
}
202-
Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
203-
}
194+
# define PyStackRef_CLOSE(stackref) \
195+
do { \
196+
_PyStackRef _close_tmp = (stackref); \
197+
if (!PyStackRef_IsDeferred(_close_tmp)) { \
198+
Py_DECREF(PyStackRef_AsPyObjectBorrow(_close_tmp)); \
199+
} \
200+
} while (0)
204201
#else
205-
# define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref));
202+
# define PyStackRef_CLOSE(stackref) Py_DECREF(PyStackRef_AsPyObjectBorrow(stackref))
206203
#endif
207204

208205
#define PyStackRef_XCLOSE(stackref) \
@@ -227,7 +224,7 @@ PyStackRef_DUP(_PyStackRef stackref)
227224
return stackref;
228225
}
229226
#else
230-
# define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)));
227+
# define PyStackRef_DUP(stackref) PyStackRef_FromPyObjectSteal(Py_NewRef(PyStackRef_AsPyObjectBorrow(stackref)))
231228
#endif
232229

233230
static inline void

‎Python/ceval.c‎

Lines changed: 46 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,27 @@
5252
# error "ceval.c must be build with Py_BUILD_CORE define for best performance"
5353
#endif
5454

55-
#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS) && !defined(Py_GIL_DISABLED)
55+
#if !defined(Py_DEBUG) && !defined(Py_TRACE_REFS)
5656
// GH-89279: The MSVC compiler does not inline these static inline functions
5757
// in PGO build in _PyEval_EvalFrameDefault(), because this function is over
5858
// the limit of PGO, and that limit cannot be configured.
5959
// Define them as macros to make sure that they are always inlined by the
6060
// preprocessor.
61-
// TODO: implement Py_DECREF macro for Py_GIL_DISABLED
61+
62+
#undef Py_IS_TYPE
63+
#define Py_IS_TYPE(ob, type) \
64+
(_PyObject_CAST(ob)->ob_type == (type))
65+
66+
#undef Py_XDECREF
67+
#define Py_XDECREF(arg) \
68+
do { \
69+
PyObject *xop = _PyObject_CAST(arg); \
70+
if (xop != NULL) { \
71+
Py_DECREF(xop); \
72+
} \
73+
} while (0)
74+
75+
#ifndef Py_GIL_DISABLED
6276

6377
#undef Py_DECREF
6478
#define Py_DECREF(arg) \
@@ -74,19 +88,6 @@
7488
} \
7589
} while (0)
7690

77-
#undef Py_XDECREF
78-
#define Py_XDECREF(arg) \
79-
do { \
80-
PyObject *xop = _PyObject_CAST(arg); \
81-
if (xop != NULL) { \
82-
Py_DECREF(xop); \
83-
} \
84-
} while (0)
85-
86-
#undef Py_IS_TYPE
87-
#define Py_IS_TYPE(ob, type) \
88-
(_PyObject_CAST(ob)->ob_type == (type))
89-
9091
#undef _Py_DECREF_SPECIALIZED
9192
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
9293
do { \
@@ -100,6 +101,36 @@
100101
d(op); \
101102
} \
102103
} while (0)
104+
105+
#else // Py_GIL_DISABLED
106+
107+
#define _DECREF(arg, DEALLOC) \
108+
do { \
109+
PyObject *op = _PyObject_CAST(arg); \
110+
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
111+
if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
112+
break; \
113+
} \
114+
_Py_DECREF_STAT_INC(); \
115+
if (_Py_IsOwnedByCurrentThread(op)) { \
116+
local--; \
117+
_Py_atomic_store_uint32_relaxed(&op->ob_ref_local, local); \
118+
if (local == 0) { \
119+
_Py_MergeZeroLocalRefcount(op); \
120+
} \
121+
} \
122+
else { \
123+
_Py_DecRefShared(op); \
124+
} \
125+
} while (0)
126+
127+
#undef Py_DECREF
128+
#define Py_DECREF(arg) _DECREF(arg, Py_TYPE(op)->tp_dealloc)
129+
130+
#undef _Py_DECREF_SPECIALIZED
131+
#define _Py_DECREF_SPECIALIZED(arg, dealloc) _DECREF(arg, (destructor)(dealloc))
132+
133+
#endif
103134
#endif
104135

105136

0 commit comments

Comments
 (0)