Fix object_visits calculation

Author: sergey-miryanov

Python/gc.c

@@ -525,12 +525,18 @@ update_refs(PyGC_Head *containers)
     return candidates;
 }
 
+struct visit_decref_context {
+    PyObject *parent;
+    struct gc_generation_stats *stats;
+};
+
 /* A traversal callback for subtract_refs. */
 static int
-visit_decref(PyObject *op, void *parent)
+visit_decref(PyObject *op, void *arg)
 {
-    OBJECT_STAT_INC(object_visits);
-    _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op));
+    struct visit_decref_context *ctx = (struct visit_decref_context *)arg;
+    ctx->stats->object_visits += 1;
+    _PyObject_ASSERT(ctx->parent, !_PyObject_IsFreed(op));
 
     if (_PyObject_IS_GC(op)) {
         PyGC_Head *gc = AS_GC(op);
@@ -577,25 +583,35 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg)
  * reachable from outside containers, and so can't be collected.
  */
 static void
-subtract_refs(PyGC_Head *containers)
+subtract_refs(PyGC_Head *containers, struct gc_generation_stats *stats)
 {
     traverseproc traverse;
     PyGC_Head *gc = GC_NEXT(containers);
     for (; gc != containers; gc = GC_NEXT(gc)) {
         PyObject *op = FROM_GC(gc);
         traverse = Py_TYPE(op)->tp_traverse;
+        struct visit_decref_context ctx = {
+            .parent = op,
+            .stats = stats
+        };
         (void) traverse(op,
                         visit_decref,
-                        op);
+                        &ctx);
     }
 }
 
+struct visit_reachable_context {
+    PyGC_Head *head;
+    struct gc_generation_stats *stats;
+};
+
 /* A traversal callback for move_unreachable. */
 static int
 visit_reachable(PyObject *op, void *arg)
 {
-    PyGC_Head *reachable = arg;
-    OBJECT_STAT_INC(object_visits);
+    struct visit_reachable_context *ctx = (struct visit_reachable_context *)arg;
+    ctx->stats->object_visits += 1;
+    PyGC_Head *reachable = ctx->head;
     if (!_PyObject_IS_GC(op)) {
         return 0;
     }
@@ -667,7 +683,7 @@ visit_reachable(PyObject *op, void *arg)
  * So we can not gc_list_* functions for unreachable until we remove the flag.
  */
 static void
-move_unreachable(PyGC_Head *young, PyGC_Head *unreachable)
+move_unreachable(PyGC_Head *young, PyGC_Head *unreachable, struct gc_generation_stats *stats)
 {
     // previous elem in the young list, used for restore gc_prev.
     PyGC_Head *prev = young;
@@ -682,6 +698,11 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable)
      * or to the right have been scanned yet.
      */
 
+    struct visit_reachable_context ctx = {
+        .head = young,
+        .stats = stats
+    };
+
     validate_consistent_old_space(young);
     /* Record which old space we are in, and set NEXT_MASK_UNREACHABLE bit for convenience */
     uintptr_t flags = NEXT_MASK_UNREACHABLE | (gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1);
@@ -703,7 +724,7 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable)
             // young->_gc_prev == gc.  Don't do gc = GC_NEXT(gc) before!
             (void) traverse(op,
                     visit_reachable,
-                    (void *)young);
+                    &ctx);
             // relink gc_prev to prev element.
             _PyGCHead_SET_PREV(gc, prev);
             // gc is not COLLECTING state after here.
@@ -831,8 +852,9 @@ clear_unreachable_mask(PyGC_Head *unreachable)
 static int
 visit_move(PyObject *op, void *arg)
 {
-    PyGC_Head *tolist = arg;
-    OBJECT_STAT_INC(object_visits);
+    struct visit_reachable_context *ctx = (struct visit_reachable_context *)arg;
+    PyGC_Head *tolist = ctx->head;
+    ctx->stats->object_visits += 1;
     if (_PyObject_IS_GC(op)) {
         PyGC_Head *gc = AS_GC(op);
         if (gc_is_collecting(gc)) {
@@ -847,16 +869,20 @@ visit_move(PyObject *op, void *arg)
  * into finalizers set.
  */
 static void
-move_legacy_finalizer_reachable(PyGC_Head *finalizers)
+move_legacy_finalizer_reachable(PyGC_Head *finalizers, struct gc_generation_stats *stats)
 {
+    struct visit_reachable_context ctx = {
+        .head = finalizers,
+        .stats = stats
+    };
     traverseproc traverse;
     PyGC_Head *gc = GC_NEXT(finalizers);
     for (; gc != finalizers; gc = GC_NEXT(gc)) {
         /* Note that the finalizers list may grow during this. */
         traverse = Py_TYPE(FROM_GC(gc))->tp_traverse;
         (void) traverse(FROM_GC(gc),
                         visit_move,
-                        (void *)finalizers);
+                        &ctx);
     }
 }
 
@@ -1244,15 +1270,15 @@ flag is cleared (for example, by using 'clear_unreachable_mask' function or
 by a call to 'move_legacy_finalizers'), the 'unreachable' list is not a normal
 list and we can not use most gc_list_* functions for it. */
 static inline Py_ssize_t
-deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) {
+deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable, struct gc_generation_stats *stats) {
     validate_list(base, collecting_clear_unreachable_clear);
     /* Using ob_refcnt and gc_refs, calculate which objects in the
      * container set are reachable from outside the set (i.e., have a
      * refcount greater than 0 when all the references within the
      * set are taken into account).
      */
     Py_ssize_t candidates = update_refs(base);  // gc_prev is used for gc_refs
-    subtract_refs(base);
+    subtract_refs(base, stats);
 
     /* Leave everything reachable from outside base in base, and move
      * everything else (in base) to unreachable.
@@ -1289,7 +1315,7 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) {
      * the reachable objects instead.  But this is a one-time cost, probably not
      * worth complicating the code to speed just a little.
      */
-    move_unreachable(base, unreachable);  // gc_prev is pointer again
+    move_unreachable(base, unreachable, stats);  // gc_prev is pointer again
     validate_list(base, collecting_clear_unreachable_clear);
     validate_list(unreachable, collecting_set_unreachable_set);
     return candidates;
@@ -1310,7 +1336,8 @@ PREV_MARK_COLLECTING set, but the objects in this set are going to be removed so
 we can skip the expense of clearing the flag to avoid extra iteration. */
 static inline void
 handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable,
-                           PyGC_Head *old_generation)
+                           PyGC_Head *old_generation,
+                           struct gc_generation_stats *stats)
 {
     // Remove the PREV_MASK_COLLECTING from unreachable
     // to prepare it for a new call to 'deduce_unreachable'
@@ -1320,7 +1347,7 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable,
     // have the PREV_MARK_COLLECTING set, but the objects are going to be
     // removed so we can skip the expense of clearing the flag.
     PyGC_Head* resurrected = unreachable;
-    deduce_unreachable(resurrected, still_unreachable);
+    deduce_unreachable(resurrected, still_unreachable, stats);
     clear_unreachable_mask(still_unreachable);
 
     // Move the resurrected objects to the old generation for future collection.
@@ -1432,14 +1459,15 @@ struct container_and_flag {
     PyGC_Head *container;
     int visited_space;
     intptr_t size;
+    struct gc_generation_stats *stats;
 };
 
 /* A traversal callback for adding to container) */
 static int
 visit_add_to_container(PyObject *op, void *arg)
 {
-    OBJECT_STAT_INC(object_visits);
     struct container_and_flag *cf = (struct container_and_flag *)arg;
+    cf->stats->object_visits += 1;
     int visited = cf->visited_space;
     assert(visited == get_gc_state()->visited_space);
     if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) {
@@ -1455,12 +1483,16 @@ visit_add_to_container(PyObject *op, void *arg)
 }
 
 static intptr_t
-expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate)
+expand_region_transitively_reachable(PyGC_Head *container,
+                                     PyGC_Head *gc,
+                                     GCState *gcstate,
+                                     struct gc_generation_stats *stats)
 {
     struct container_and_flag arg = {
         .container = container,
         .visited_space = gcstate->visited_space,
-        .size = 0
+        .size = 0,
+        .stats = stats
     };
     assert(GC_NEXT(gc) == container);
     while (gc != container) {
@@ -1529,13 +1561,14 @@ move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space)
 }
 
 static intptr_t
-mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space)
+mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space, struct gc_generation_stats *stats)
 {
     // Transitively traverse all objects from reachable, until empty
     struct container_and_flag arg = {
         .container = reachable,
         .visited_space = visited_space,
-        .size = 0
+        .size = 0,
+        .stats = stats
     };
     while (!gc_list_is_empty(reachable)) {
         PyGC_Head *gc = _PyGCHead_NEXT(reachable);
@@ -1552,7 +1585,7 @@ mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space)
 }
 
 static intptr_t
-mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start)
+mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start, struct gc_generation_stats *stats)
 {
     PyGC_Head reachable;
     gc_list_init(&reachable);
@@ -1605,13 +1638,13 @@ mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, b
         ts = PyThreadState_Next(ts);
         HEAD_UNLOCK(runtime);
     }
-    objects_marked += mark_all_reachable(&reachable, visited, visited_space);
+    objects_marked += mark_all_reachable(&reachable, visited, visited_space, stats);
     assert(gc_list_is_empty(&reachable));
     return objects_marked;
 }
 
 static intptr_t
-mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space)
+mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, struct gc_generation_stats *stats)
 {
     PyGC_Head reachable;
     gc_list_init(&reachable);
@@ -1628,19 +1661,19 @@ mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_sp
         objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space);
         objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space);
     }
-    objects_marked += mark_all_reachable(&reachable, visited, visited_space);
+    objects_marked += mark_all_reachable(&reachable, visited, visited_space, stats);
     assert(gc_list_is_empty(&reachable));
     return objects_marked;
 }
 
 static intptr_t
-mark_at_start(PyThreadState *tstate)
+mark_at_start(PyThreadState *tstate, struct gc_generation_stats *stats)
 {
     // TO DO -- Make this incremental
     GCState *gcstate = &tstate->interp->gc;
     PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head;
-    Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space);
-    objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true);
+    Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space, stats);
+    objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true, stats);
     gcstate->work_to_do -= objects_marked;
     gcstate->phase = GC_PHASE_COLLECT;
     validate_spaces(gcstate);
@@ -1686,7 +1719,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_generation_stats *stats)
     }
     untrack_tuples(&gcstate->young.head);
     if (gcstate->phase == GC_PHASE_MARK) {
-        Py_ssize_t objects_marked = mark_at_start(tstate);
+        Py_ssize_t objects_marked = mark_at_start(tstate, stats);
         stats->objects_transitively_reachable += objects_marked;
         stats->candidates += objects_marked;
         gcstate->work_to_do -= objects_marked;
@@ -1701,7 +1734,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_generation_stats *stats)
     if (scale_factor < 2) {
         scale_factor = 2;
     }
-    intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false);
+    intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false, stats);
     stats->objects_transitively_reachable += objects_marked;
     gcstate->work_to_do -= objects_marked;
     gc_list_set_space(&gcstate->young.head, gcstate->visited_space);
@@ -1717,7 +1750,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_generation_stats *stats)
         increment_size++;
         assert(!_Py_IsImmortal(FROM_GC(gc)));
         gc_set_old_space(gc, gcstate->visited_space);
-        increment_size += expand_region_transitively_reachable(&increment, gc, gcstate);
+        increment_size += expand_region_transitively_reachable(&increment, gc, gcstate, stats);
     }
     stats->objects_not_transitively_reachable += increment_size;
     validate_list(&increment, collecting_clear_unreachable_clear);
@@ -1781,7 +1814,7 @@ gc_collect_region(PyThreadState *tstate,
     assert(!_PyErr_Occurred(tstate));
 
     gc_list_init(&unreachable);
-    stats->candidates = deduce_unreachable(from, &unreachable);
+    stats->candidates = deduce_unreachable(from, &unreachable, stats);
     validate_consistent_old_space(from);
     untrack_tuples(from);
 
@@ -1803,7 +1836,7 @@ gc_collect_region(PyThreadState *tstate,
      * unreachable objects reachable *from* those are also uncollectable,
      * and we move those into the finalizers list too.
      */
-    move_legacy_finalizer_reachable(&finalizers);
+    move_legacy_finalizer_reachable(&finalizers, stats);
     validate_list(&finalizers, collecting_clear_unreachable_clear);
     validate_list(&unreachable, collecting_set_unreachable_clear);
     /* Print debugging information. */
@@ -1826,7 +1859,7 @@ gc_collect_region(PyThreadState *tstate,
      * objects that are still unreachable */
     PyGC_Head final_unreachable;
     gc_list_init(&final_unreachable);
-    handle_resurrected_objects(&unreachable, &final_unreachable, to);
+    handle_resurrected_objects(&unreachable, &final_unreachable, to, stats);
 
     /* Clear weakrefs to objects in the unreachable set.  See the comments
      * above handle_weakref_callbacks() for details.