Skip to content

GH-124567: Reduce overhead of cycle GC. #124717

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
markshannon wants to merge 3 commits into from
Closed

GH-124567: Reduce overhead of cycle GC. #124717

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9d28e99
Attempt to match 3.12 in work done in GC
markshannon Sep 26, 2024
d451ace
Lower scaling factor
markshannon Sep 27, 2024
615a96e
Pre-scan sys.modules
markshannon Sep 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Pre-scan sys.modules
  • Loading branch information
@markshannon
markshannon committed Sep 27, 2024
commit 615a96efb9fe66d8c2bdb9603b29247ba36ac5a1
3 changes: 2 additions & 1 deletion Include/internal/pycore_gc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -328,7 +328,8 @@ struct _gc_runtime_state {
Py_ssize_t heap_size;
Py_ssize_t work_to_do;
/* Which of the old spaces is the visited space */
int visited_space;
uint8_t visited_space;
uint8_t scan_reachable;

#ifdef Py_GIL_DISABLED
/* This is the number of objects that survived the last full
Expand Down
1 change: 1 addition & 0 deletions Lib/test/test_gc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1115,6 +1115,7 @@ def make_ll(depth):
olds.append(newhead)
if len(olds) == 20:
live = _testinternalcapi.get_heap_size()
print(i, live, baseline_live)
self.assertLess(live, baseline_live*2)
del olds[:]
if not enabled:
Expand Down
32 changes: 29 additions & 3 deletions Python/gc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1287,8 +1287,8 @@ gc_list_set_space(PyGC_Head *list, int space)
*/

/* Multiply by 5, so that the default incremental threshold of 10
* scans objects at the same rate as the young generation */
#define SCAN_RATE_MULTIPLIER 10
* scans objects at half the rate of the young generation */
#define SCAN_RATE_MULTIPLIER 5

static void
add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats)
Expand Down Expand Up @@ -1428,8 +1428,30 @@ completed_cycle(GCState *gcstate)
gc = next;
}
gcstate->work_to_do = 0;
gcstate->scan_reachable = 1;
}


static void
gc_mark_reachable(PyThreadState *tstate)
{
GCState *gcstate = &tstate->interp->gc;
PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head;
PyObject *sysdict = tstate->interp->sysdict;
PyObject *sysmod = PyDict_GetItemString(sysdict, "modules");
if (sysmod == NULL) {
return;
}
PyGC_Head reachable;
gc_list_init(&reachable);
PyGC_Head *gc = _Py_AS_GC(sysmod);
gc_list_move(gc, &reachable);
gc_set_old_space(gc, gcstate->visited_space);
gcstate->work_to_do -= expand_region_transitively_reachable(&reachable, gc, gcstate);
gc_list_merge(&reachable, visited);
}


static void
gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
{
Expand All @@ -1439,6 +1461,10 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head;
PyGC_Head increment;
gc_list_init(&increment);
if (gcstate->scan_reachable) {
gc_mark_reachable(tstate);
gcstate->scan_reachable = 0;
}
Py_ssize_t scale_factor = gcstate->old[0].threshold;
if (scale_factor < 1) {
scale_factor = 1;
Expand All @@ -1461,7 +1487,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
gc_list_validate_space(&survivors, gcstate->visited_space);
gc_list_merge(&survivors, visited);
assert(gc_list_is_empty(&increment));
Py_ssize_t delta = (gcstate->heap_size - gcstate->prior_heap_size)*2;
Py_ssize_t delta = (gcstate->heap_size - gcstate->prior_heap_size)*3;

@rruuaanng rruuaanng Sep 29, 2024

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Binary operations, where two operands require spaces, may be more standard.
(gcstate->heap_size - gcstate->prior_heap_size) * 3

delta += gcstate->young.threshold * SCAN_RATE_MULTIPLIER / scale_factor;
if (delta > 0) {
gcstate->work_to_do += delta;
Expand Down