Revert "GH-117108: Change the size of the GC increment to about 1% of…

… the total heap size. (GH-117120)" This reverts commit e28477f.
python · Yhg1s · Sep 30, 2024 · Sep 29, 2024 · Sep 29, 2024 · Sep 29, 2024
commit 60a248bdfc406558216367bf0eda0422746914ef
diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
@@ -315,7 +315,6 @@ struct _gc_runtime_state {
     /* a list of callbacks to be invoked when collection is performed */
     PyObject *callbacks;
 
-    Py_ssize_t heap_size;
     Py_ssize_t work_to_do;
     /* Which of the old spaces is the visited space */
     int visited_space;
@@ -363,7 +362,7 @@ extern void _PyGC_Unfreeze(PyInterpreterState *interp);
 /* Number of frozen objects */
 extern Py_ssize_t _PyGC_GetFreezeCount(PyInterpreterState *interp);
 
-extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, int generation);
+extern PyObject *_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation);
 extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs);
 
 // Functions to clear types free lists

diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py
@@ -1122,31 +1122,28 @@ def make_ll(depth):
                 head = LinkedList(head, head.prev)
             return head
 
-        head = make_ll(1000)
-        count = 1000
+        head = make_ll(10000)
+        count = 10000
 
-        # There will be some objects we aren't counting,
-        # e.g. the gc stats dicts. This test checks
-        # that the counts don't grow, so we try to
-        # correct for the uncounted objects
-        # This is just an estimate.
-        CORRECTION = 20
+        # We expect the counts to go negative eventually
+        # as there will some objects we aren't counting,
+        # e.g. the gc stats dicts. The test merely checks
+        # that the counts don't grow.
 
         enabled = gc.isenabled()
         gc.enable()
         olds = []
-        for i in range(20_000):
-            newhead = make_ll(20)
-            count += 20
+        for i in range(1000):
+            newhead = make_ll(200)
+            count += 200
             newhead.surprise = head
             olds.append(newhead)
-            if len(olds) == 20:
+            if len(olds) == 50:
                 stats = gc.get_stats()
                 young = stats[0]
                 incremental = stats[1]
                 old = stats[2]
                 collected = young['collected'] + incremental['collected'] + old['collected']
-                count += CORRECTION
                 live = count - collected
                 self.assertLess(live, 25000)
                 del olds[:]

@@ -326,7 +326,7 @@ gc_get_objects_impl(PyObject *module, Py_ssize_t generation)
     }
 
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    return _PyGC_GetObjects(interp, (int)generation);
+    return _PyGC_GetObjects(interp, generation);
 }
 
 /*[clinic input]

diff --git a/Python/gc.c b/Python/gc.c
@@ -181,7 +181,6 @@ _PyGC_Init(PyInterpreterState *interp)
     if (gcstate->callbacks == NULL) {
         return _PyStatus_NO_MEMORY();
     }
-    gcstate->heap_size = 0;
 
     return _PyStatus_OK();
 }
@@ -1232,7 +1231,7 @@ gc_collect_region(PyThreadState *tstate,
                   struct gc_collection_stats *stats);
 
 static inline Py_ssize_t
-gc_list_set_space(PyGC_Head *list, int space)
+gc_list_set_space(PyGC_Head *list, uintptr_t space)
 {
     Py_ssize_t size = 0;
     PyGC_Head *gc;
@@ -1258,9 +1257,9 @@ gc_list_set_space(PyGC_Head *list, int space)
  * N == 1.4 (1 + 4/threshold)
  */
 
-/* Divide by 10, so that the default incremental threshold of 10
- * scans objects at 1% of the heap size */
-#define SCAN_RATE_DIVISOR 10
+/* Multiply by 4 so that the default incremental threshold of 10
+ * scans objects at 20% the rate of object creation */
+#define SCAN_RATE_MULTIPLIER 2
 
 static void
 add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats)
@@ -1312,7 +1311,7 @@ gc_collect_young(PyThreadState *tstate,
     if (scale_factor < 1) {
         scale_factor = 1;
     }
-    gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor;
+    gcstate->work_to_do += survivor_count + survivor_count * SCAN_RATE_MULTIPLIER / scale_factor;
     add_stats(gcstate, 0, stats);
 }
 
@@ -1383,12 +1382,12 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat
 static void
 completed_cycle(GCState *gcstate)
 {
-#ifdef Py_DEBUG
     PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head;
     assert(gc_list_is_empty(not_visited));
-#endif
     gcstate->visited_space = flip_old_space(gcstate->visited_space);
-    gcstate->work_to_do = 0;
+    if (gcstate->work_to_do > 0) {
+        gcstate->work_to_do = 0;
+    }
 }
 
 static void
@@ -1403,13 +1402,13 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
     if (scale_factor < 1) {
         scale_factor = 1;
     }
+    Py_ssize_t increment_size = 0;
     gc_list_merge(&gcstate->young.head, &increment);
     gcstate->young.count = 0;
     if (gcstate->visited_space) {
         /* objects in visited space have bit set, so we set it here */
         gc_list_set_space(&increment, 1);
     }
-    Py_ssize_t increment_size = 0;
     while (increment_size < gcstate->work_to_do) {
         if (gc_list_is_empty(not_visited)) {
             break;
@@ -1423,11 +1422,14 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
     PyGC_Head survivors;
     gc_list_init(&survivors);
     gc_collect_region(tstate, &increment, &survivors, UNTRACK_TUPLES, stats);
+    Py_ssize_t survivor_count = gc_list_size(&survivors);
     gc_list_merge(&survivors, visited);
     assert(gc_list_is_empty(&increment));
-    gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor;
+    gcstate->work_to_do += survivor_count + survivor_count * SCAN_RATE_MULTIPLIER / scale_factor;
     gcstate->work_to_do -= increment_size;
-
+    if (gcstate->work_to_do < 0) {
+        gcstate->work_to_do = 0;
+    }
     validate_old(gcstate);
     add_stats(gcstate, 1, stats);
     if (gc_list_is_empty(not_visited)) {
@@ -1673,7 +1675,7 @@ _PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs)
 }
 
 PyObject *
-_PyGC_GetObjects(PyInterpreterState *interp, int generation)
+_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation)
 {
     assert(generation >= -1 && generation < NUM_GENERATIONS);
     GCState *gcstate = &interp->gc;
@@ -1992,7 +1994,6 @@ _PyObject_GC_Link(PyObject *op)
     gc->_gc_next = 0;
     gc->_gc_prev = 0;
     gcstate->young.count++; /* number of allocated GC objects */
-    gcstate->heap_size++;
     if (gcstate->young.count > gcstate->young.threshold &&
         gcstate->enabled &&
         gcstate->young.threshold &&
@@ -2119,7 +2120,6 @@ PyObject_GC_Del(void *op)
     if (gcstate->young.count > 0) {
         gcstate->young.count--;
     }
-    gcstate->heap_size--;
     PyObject_Free(((char *)op)-presize);
 }
 

diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c
@@ -1377,7 +1377,7 @@ visit_get_objects(const mi_heap_t *heap, const mi_heap_area_t *area,
 }
 
 PyObject *
-_PyGC_GetObjects(PyInterpreterState *interp, int generation)
+_PyGC_GetObjects(PyInterpreterState *interp, Py_ssize_t generation)
 {
     PyObject *result = PyList_New(0);
     if (!result) {