Skip to content

Commit 149ea9d

Browse files
colesburycolesbury
committed
Deferred reference counting
Initial partial implementation of deferred reference counting.
1 parent 9c1f7ba commit 149ea9d

9 files changed

Lines changed: 110 additions & 29 deletions

File tree

‎Include/internal/pycore_object.h‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,34 @@ _PyObject_SetMaybeWeakref(PyObject *op)
360360
}
361361
}
362362

363+
/* Marks the object as support deferred reference counting.
364+
*
365+
* The object's type must be GC-enabled. This function is not thread-safe with
366+
* respect to concurrent modifications; it must be called before the object
367+
* becomes visible to other threads.
368+
*
369+
* Deferred refcounted objects are marked as "queued" to prevent merging
370+
* reference count fields outside the garbage collector.
371+
*/
372+
static inline void
373+
_PyObject_SetDeferredRefcount(PyObject *op)
374+
{
375+
assert(_Py_ThreadLocal(op) && "non thread-safe");
376+
assert(!_PyObject_HasDeferredRefcount(op) && "already uses deferred refcounting");
377+
assert(PyType_IS_GC(Py_TYPE(op)));
378+
op->ob_ref_local += _Py_REF_DEFERRED_MASK + 1;
379+
op->ob_ref_shared = (op->ob_ref_shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_QUEUED;
380+
}
381+
382+
#define _PyObject_SET_DEFERRED_REFCOUNT(op) _PyObject_SetDeferredRefcount(_PyObject_CAST(op))
383+
384+
// Check is refcount is deferred or immortal
385+
static inline int
386+
_Py_REF_NON_IMMEDIATE(uint32_t local)
387+
{
388+
return _Py_STATIC_CAST(int32_t, local) <= Py_REF_IMMORTAL;
389+
}
390+
363391
#ifdef Py_REF_DEBUG
364392
extern void _PyDebug_PrintTotalRefs(void);
365393
#endif

‎Include/object.h‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y);
135135
#define Py_Is(x, y) ((x) == (y))
136136

137137
static inline void
138-
_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal);
138+
_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal, int *deferred);
139139

140140
static inline void
141141
_PyRef_UnpackShared(uint32_t bits, Py_ssize_t *refcount, int *queued, int *merged);
@@ -148,7 +148,7 @@ static inline Py_ssize_t Py_REFCNT(PyObject *ob) {
148148
int immortal;
149149

150150
uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local);
151-
_PyRef_UnpackLocal(local, &local_refcount, &immortal);
151+
_PyRef_UnpackLocal(local, &local_refcount, &immortal, NULL);
152152

153153
if (immortal) {
154154
return 999;
@@ -617,7 +617,7 @@ _Py_ThreadLocal(PyObject *op)
617617
#define _Py_REF_LOCAL_SHIFT 0
618618
#define _Py_REF_LOCAL_INIT 1
619619
#define Py_REF_IMMORTAL -1
620-
#define _Py_REF_DEFERRED_MASK 0x40000000
620+
#define _Py_REF_DEFERRED_MASK 0xC0000000
621621

622622
// 30 2
623623
// [refcount] [bits]
@@ -884,10 +884,17 @@ static inline PyObject* _Py_XNewRef(PyObject *obj)
884884
#endif
885885

886886
static inline void
887-
_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal)
887+
_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal, int *deferred)
888888
{
889889
*refcount = bits >> _Py_REF_LOCAL_SHIFT;
890890
*immortal = _Py_REF_IS_IMMORTAL(bits);
891+
int is_deferred = _Py_STATIC_CAST(int32_t, bits) < Py_REF_IMMORTAL;
892+
if (is_deferred) {
893+
*refcount -= _Py_REF_DEFERRED_MASK;
894+
}
895+
if (deferred) {
896+
*deferred = is_deferred;
897+
}
891898
}
892899

893900
static inline void
@@ -925,7 +932,7 @@ _PyObject_IsReferened(PyObject *op)
925932
int immortal;
926933

927934
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
928-
_PyRef_UnpackLocal(local, &local_refcount, &immortal);
935+
_PyRef_UnpackLocal(local, &local_refcount, &immortal, NULL);
929936

930937
if (immortal) {
931938
return 1;
@@ -940,6 +947,13 @@ _PyObject_IsReferened(PyObject *op)
940947
return (local_refcount + shared_refcount) > 0;
941948
}
942949

950+
static inline int
951+
_PyObject_HasDeferredRefcount(PyObject *op)
952+
{
953+
uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
954+
return _Py_STATIC_CAST(int32_t, local) < Py_REF_IMMORTAL;
955+
}
956+
943957
static inline void
944958
_Py_RESURRECT(PyObject *op, Py_ssize_t refcnt)
945959
{

‎Modules/gcmodule.c‎

Lines changed: 44 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -318,13 +318,13 @@ static Py_ssize_t
318318
_Py_GC_REFCNT(PyObject *op)
319319
{
320320
Py_ssize_t local, shared;
321-
int immortal;
321+
int immortal, deferred;
322322

323-
_PyRef_UnpackLocal(op->ob_ref_local, &local, &immortal);
323+
_PyRef_UnpackLocal(op->ob_ref_local, &local, &immortal, &deferred);
324324
_PyRef_UnpackShared(op->ob_ref_shared, &shared, NULL, NULL);
325325
assert(!immortal);
326326

327-
return local + shared;
327+
return local + shared - deferred;
328328
}
329329

330330
typedef int (gc_visit_fn)(PyGC_Head* gc, void *arg);
@@ -682,9 +682,35 @@ find_dead_shared_keys(_PyObjectQueue **queue, int *num_unmarked)
682682
}
683683
}
684684

685+
static void
686+
merge_refcount(PyObject *op, Py_ssize_t extra)
687+
{
688+
Py_ssize_t local_refcount, shared_refcount;
689+
int immortal, deferred;
690+
691+
assert(_PyRuntime.stop_the_world);
692+
693+
_PyRef_UnpackLocal(op->ob_ref_local, &local_refcount, &immortal, &deferred);
694+
_PyRef_UnpackShared(op->ob_ref_shared, &shared_refcount, NULL, NULL);
695+
assert(!immortal && "immortal objects should not be in garbage");
696+
697+
Py_ssize_t refcount = local_refcount + shared_refcount;
698+
refcount += extra;
699+
refcount -= deferred;
700+
701+
#ifdef Py_REF_DEBUG
702+
_Py_IncRefTotalN(extra);
703+
#endif
704+
705+
op->ob_tid = 0;
706+
op->ob_ref_local = 0;
707+
op->ob_ref_shared = _Py_REF_PACK_SHARED(refcount, _Py_REF_MERGED);
708+
}
709+
685710
struct update_refs_args {
686711
PyGC_Head *list;
687712
int split_keys_marked;
713+
_PyGC_Reason gc_reason;
688714
};
689715

690716
// Compute the number of external references to objects in the heap
@@ -730,6 +756,15 @@ update_refs(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size
730756
}
731757
}
732758

759+
if (arg->gc_reason == GC_REASON_SHUTDOWN) {
760+
if (_PyObject_HasDeferredRefcount(op)) {
761+
// Disable deferred reference counting when we're shutting down.
762+
// This is useful for interp->sysdict because the last reference
763+
// to it is cleared after the last GC cycle.
764+
merge_refcount(op, 0);
765+
}
766+
}
767+
733768
// Add the actual refcount to gc_refs.
734769
Py_ssize_t refcount = _Py_GC_REFCNT(op);
735770
_PyObject_ASSERT(op, refcount >= 0);
@@ -1008,23 +1043,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers)
10081043
static void
10091044
incref_merge(PyObject *op)
10101045
{
1011-
Py_ssize_t local_refcount, shared_refcount;
1012-
int immortal;
1013-
1014-
assert(_PyRuntime.stop_the_world);
1015-
1016-
#ifdef Py_REF_DEBUG
1017-
_Py_IncRefTotal();
1018-
#endif
1019-
1020-
_PyRef_UnpackLocal(op->ob_ref_local, &local_refcount, &immortal);
1021-
_PyRef_UnpackShared(op->ob_ref_shared, &shared_refcount, NULL, NULL);
1022-
assert(!immortal && "immortal objects should not be in garbage");
1023-
1024-
Py_ssize_t refcount = local_refcount + shared_refcount + 1;
1025-
op->ob_tid = 0;
1026-
op->ob_ref_local = 0;
1027-
op->ob_ref_shared = _Py_REF_PACK_SHARED(refcount, _Py_REF_MERGED);
1046+
merge_refcount(op, 1);
10281047
}
10291048

10301049
/* Subtracts one from the refcount field. */
@@ -1594,7 +1613,11 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
15941613
validate_tracked_heap(_PyGC_PREV_MASK|_PyGC_PREV_MASK_UNREACHABLE, 0);
15951614

15961615
gc_list_init(&young);
1597-
struct update_refs_args args = { .list = &young, .split_keys_marked = 0 };
1616+
struct update_refs_args args = {
1617+
.list = &young,
1618+
.split_keys_marked = 0,
1619+
.gc_reason = reason,
1620+
};
15981621
visit_heaps2(mi_heap_tag_gc, update_refs, &args);
15991622

16001623
_PyObjectQueue *dead_keys = NULL;

‎Objects/descrobject.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -903,6 +903,7 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name)
903903

904904
descr = (PyDescrObject *)PyType_GenericAlloc(descrtype, 0);
905905
if (descr != NULL) {
906+
_PyObject_SET_DEFERRED_REFCOUNT(descr);
906907
descr->d_type = (PyTypeObject*)Py_XNewRef(type);
907908
descr->d_name = PyUnicode_InternFromString(name);
908909
if (descr->d_name == NULL) {

‎Objects/dictobject.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1371,6 +1371,10 @@ _PyDict_MaybeUntrack(PyObject *op)
13711371
if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op))
13721372
return;
13731373

1374+
if (_PyObject_HasDeferredRefcount(op)) {
1375+
return;
1376+
}
1377+
13741378
mp = (PyDictObject *) op;
13751379
numentries = mp->ma_keys->dk_nentries;
13761380
if (_PyDict_HasSplitTable(mp)) {

‎Objects/funcobject.c‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,9 @@ PyFunction_ClearWatcher(int watcher_id)
7777
PyFunctionObject *
7878
_PyFunction_FromConstructor(PyFrameConstructor *constr)
7979
{
80-
80+
// NOTE(sgross): functions created via FrameConstructor do *not* use
81+
// deferred reference counting because they are typically not part of cycles
82+
// nor accessed by multiple threads.
8183
PyFunctionObject *op = PyObject_GC_New(PyFunctionObject, &PyFunction_Type);
8284
if (op == NULL) {
8385
return NULL;
@@ -172,6 +174,9 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname
172174
op->func_annotations = NULL;
173175
op->vectorcall = _PyFunction_Vectorcall;
174176
op->func_version = 0;
177+
if ((code_obj->co_flags & CO_NESTED) == 0) {
178+
_PyObject_SET_DEFERRED_REFCOUNT(op);
179+
}
175180
_PyObject_GC_TRACK(op);
176181
handle_func_event(PyFunction_EVENT_CREATE, op, NULL);
177182
return (PyObject *)op;

‎Objects/moduleobject.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,9 @@ new_module_notrack(PyTypeObject *mt)
107107
m->md_name = NULL;
108108
m->md_dict = PyDict_New();
109109
if (m->md_dict != NULL) {
110+
PyObject_GC_Track(m->md_dict);
111+
_PyObject_SET_DEFERRED_REFCOUNT(m->md_dict);
112+
_PyObject_SET_DEFERRED_REFCOUNT(m);
110113
return m;
111114
}
112115
Py_DECREF(m);

‎Objects/typeobject.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2887,6 +2887,7 @@ type_new_alloc(type_new_ctx *ctx)
28872887
if (type == NULL) {
28882888
return NULL;
28892889
}
2890+
_PyObject_SET_DEFERRED_REFCOUNT(type);
28902891
PyHeapTypeObject *et = (PyHeapTypeObject *)type;
28912892

28922893
// Initialize tp_flags.
@@ -3775,6 +3776,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module,
37753776
if (res == NULL) {
37763777
goto finally;
37773778
}
3779+
_PyObject_SET_DEFERRED_REFCOUNT(res);
37783780
res_start = (char*)res;
37793781

37803782
type = &res->ht_type;

‎Python/sysmodule.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1876,16 +1876,17 @@ sys_getfullrefcount(PyObject *module, PyObject *object)
18761876
uintptr_t tid = _PyObject_ThreadId(object);
18771877

18781878
Py_ssize_t local, shared;
1879-
int immortal, queued, merged;
1879+
int immortal, deferred, queued, merged;
18801880

1881-
_PyRef_UnpackLocal(object->ob_ref_local, &local, &immortal);
1881+
_PyRef_UnpackLocal(object->ob_ref_local, &local, &immortal, &deferred);
18821882
_PyRef_UnpackShared(object->ob_ref_shared, &shared, &queued, &merged);
18831883

18841884
PyDict_SetItemString(res, "local", PyLong_FromSsize_t(local));
18851885
PyDict_SetItemString(res, "shared", PyLong_FromSsize_t(shared));
18861886
PyDict_SetItemString(res, "merged", PyBool_FromLong(merged));
18871887
PyDict_SetItemString(res, "queued", PyBool_FromLong(queued));
18881888
PyDict_SetItemString(res, "immortal", PyBool_FromLong(immortal));
1889+
PyDict_SetItemString(res, "deferred", PyBool_FromLong(deferred));
18891890
if (queued) {
18901891
Py_INCREF(Py_None);
18911892
PyDict_SetItemString(res, "tid", Py_None);

0 commit comments

Comments
 (0)