Skip to content

Commit 37c3d44

Browse files
sergey-miryanovsergey-miryanov
committed
Extend statics in the GC
1 parent 1384f02 commit 37c3d44

3 files changed

Lines changed: 65 additions & 37 deletions

File tree

‎Include/internal/pycore_interp_structs.h‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -177,31 +177,32 @@ struct gc_generation {
177177
generations */
178178
};
179179

180-
struct gc_collection_stats {
181-
/* number of collected objects */
182-
Py_ssize_t collected;
183-
/* total number of uncollectable objects (put into gc.garbage) */
184-
Py_ssize_t uncollectable;
185-
// Total number of objects considered for collection and traversed:
186-
Py_ssize_t candidates;
187-
// Duration of the collection in seconds:
188-
double duration;
189-
};
190-
191180
/* Running stats per generation */
192181
struct gc_generation_stats {
182+
PyTime_t ts;
193183
/* total number of collections */
194184
Py_ssize_t collections;
185+
195186
/* total number of collected objects */
196187
Py_ssize_t collected;
197188
/* total number of uncollectable objects (put into gc.garbage) */
198189
Py_ssize_t uncollectable;
199190
// Total number of objects considered for collection and traversed:
200191
Py_ssize_t candidates;
201192
// Duration of the collection in seconds:
193+
194+
Py_ssize_t object_visits;
195+
Py_ssize_t objects_transitively_reachable;
196+
Py_ssize_t objects_not_transitively_reachable;
197+
202198
double duration;
203199
};
204200

201+
struct gc_generation_stats_buffer {
202+
struct gc_generation_stats items[11];
203+
int8_t index;
204+
};
205+
205206
enum _GCPhase {
206207
GC_PHASE_MARK = 0,
207208
GC_PHASE_COLLECT = 1
@@ -211,6 +212,10 @@ enum _GCPhase {
211212
signature of gc.collect and change the size of PyStats.gc_stats */
212213
#define NUM_GENERATIONS 3
213214

215+
struct gc_stats {
216+
struct gc_generation_stats_buffer gen[NUM_GENERATIONS];
217+
};
218+
214219
struct _gc_runtime_state {
215220
/* Is automatic collection enabled? */
216221
int enabled;
@@ -220,7 +225,7 @@ struct _gc_runtime_state {
220225
struct gc_generation old[2];
221226
/* a permanent generation which won't be collected */
222227
struct gc_generation permanent_generation;
223-
struct gc_generation_stats generation_stats[NUM_GENERATIONS];
228+
struct gc_stats generation_stats;
224229
/* true if we are currently running the collector */
225230
int collecting;
226231
// The frame that started the current collection. It might be NULL even when

‎Modules/gcmodule.c‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -347,8 +347,10 @@ gc_get_stats_impl(PyObject *module)
347347
/* To get consistent values despite allocations while constructing
348348
the result list, we use a snapshot of the running stats. */
349349
GCState *gcstate = get_gc_state();
350+
struct gc_generation_stats_buffer *buffer;
350351
for (i = 0; i < NUM_GENERATIONS; i++) {
351-
stats[i] = gcstate->generation_stats[i];
352+
buffer = &gcstate->generation_stats.gen[i];
353+
stats[i] = buffer->items[buffer->index];
352354
}
353355

354356
PyObject *result = PyList_New(0);

‎Python/gc.c‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1331,7 +1331,7 @@ static void
13311331
gc_collect_region(PyThreadState *tstate,
13321332
PyGC_Head *from,
13331333
PyGC_Head *to,
1334-
struct gc_collection_stats *stats);
1334+
struct gc_generation_stats *stats);
13351335

13361336
static inline Py_ssize_t
13371337
gc_list_set_space(PyGC_Head *list, int space)
@@ -1364,26 +1364,49 @@ gc_list_set_space(PyGC_Head *list, int space)
13641364
* scans objects at 1% of the heap size */
13651365
#define SCAN_RATE_DIVISOR 10
13661366

1367+
static struct gc_generation_stats *
1368+
gc_get_stats(GCState *gcstate, int gen)
1369+
{
1370+
struct gc_generation_stats_buffer *buffer = &gcstate->generation_stats.gen[gen];
1371+
buffer->index = (buffer->index + 1) % 11;
1372+
struct gc_generation_stats *stats = &buffer->items[buffer->index];
1373+
return stats;
1374+
}
1375+
1376+
static struct gc_generation_stats *
1377+
gc_get_prev_stats(GCState *gcstate, int gen)
1378+
{
1379+
struct gc_generation_stats_buffer *buffer = &gcstate->generation_stats.gen[gen];
1380+
struct gc_generation_stats *stats = &buffer->items[buffer->index];
1381+
return stats;
1382+
}
1383+
13671384
static void
1368-
add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats)
1385+
add_stats(GCState *gcstate, int gen, struct gc_generation_stats *stats)
13691386
{
1370-
gcstate->generation_stats[gen].duration += stats->duration;
1371-
gcstate->generation_stats[gen].collected += stats->collected;
1372-
gcstate->generation_stats[gen].uncollectable += stats->uncollectable;
1373-
gcstate->generation_stats[gen].candidates += stats->candidates;
1374-
gcstate->generation_stats[gen].collections += 1;
1387+
struct gc_generation_stats *prev_stats = gc_get_prev_stats(gcstate, gen);
1388+
struct gc_generation_stats *cur_stats = gc_get_stats(gcstate, gen);
1389+
1390+
cur_stats->ts = stats->ts;
1391+
cur_stats->collections = prev_stats->collections + 1;
1392+
cur_stats->object_visits = prev_stats->object_visits + stats->object_visits;
1393+
cur_stats->collected = prev_stats->collected + stats->collected;
1394+
cur_stats->objects_transitively_reachable = prev_stats->objects_transitively_reachable + stats->objects_transitively_reachable;
1395+
cur_stats->objects_not_transitively_reachable = prev_stats->objects_not_transitively_reachable + stats->objects_not_transitively_reachable;
1396+
cur_stats->uncollectable = prev_stats->uncollectable + stats->uncollectable;
1397+
cur_stats->candidates = prev_stats->candidates + stats->candidates;
1398+
cur_stats->duration = prev_stats->duration + stats->duration;
13751399
}
13761400

13771401
static void
13781402
gc_collect_young(PyThreadState *tstate,
1379-
struct gc_collection_stats *stats)
1403+
struct gc_generation_stats *stats)
13801404
{
13811405
GCState *gcstate = &tstate->interp->gc;
13821406
validate_spaces(gcstate);
13831407
PyGC_Head *young = &gcstate->young.head;
13841408
PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head;
13851409
untrack_tuples(young);
1386-
GC_STAT_ADD(0, collections, 1);
13871410

13881411
PyGC_Head survivors;
13891412
gc_list_init(&survivors);
@@ -1654,9 +1677,8 @@ assess_work_to_do(GCState *gcstate)
16541677
}
16551678

16561679
static void
1657-
gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
1680+
gc_collect_increment(PyThreadState *tstate, struct gc_generation_stats *stats)
16581681
{
1659-
GC_STAT_ADD(1, collections, 1);
16601682
GCState *gcstate = &tstate->interp->gc;
16611683
gcstate->work_to_do += assess_work_to_do(gcstate);
16621684
if (gcstate->work_to_do < 0) {
@@ -1665,9 +1687,9 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
16651687
untrack_tuples(&gcstate->young.head);
16661688
if (gcstate->phase == GC_PHASE_MARK) {
16671689
Py_ssize_t objects_marked = mark_at_start(tstate);
1668-
GC_STAT_ADD(1, objects_transitively_reachable, objects_marked);
1669-
gcstate->work_to_do -= objects_marked;
1690+
stats->objects_transitively_reachable += objects_marked;
16701691
stats->candidates += objects_marked;
1692+
gcstate->work_to_do -= objects_marked;
16711693
validate_spaces(gcstate);
16721694
return;
16731695
}
@@ -1680,7 +1702,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
16801702
scale_factor = 2;
16811703
}
16821704
intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false);
1683-
GC_STAT_ADD(1, objects_transitively_reachable, objects_marked);
1705+
stats->objects_transitively_reachable += objects_marked;
16841706
gcstate->work_to_do -= objects_marked;
16851707
gc_list_set_space(&gcstate->young.head, gcstate->visited_space);
16861708
gc_list_merge(&gcstate->young.head, &increment);
@@ -1697,7 +1719,7 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
16971719
gc_set_old_space(gc, gcstate->visited_space);
16981720
increment_size += expand_region_transitively_reachable(&increment, gc, gcstate);
16991721
}
1700-
GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size);
1722+
stats->objects_not_transitively_reachable += increment_size;
17011723
validate_list(&increment, collecting_clear_unreachable_clear);
17021724
gc_list_validate_space(&increment, gcstate->visited_space);
17031725
PyGC_Head survivors;
@@ -1715,9 +1737,8 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats)
17151737

17161738
static void
17171739
gc_collect_full(PyThreadState *tstate,
1718-
struct gc_collection_stats *stats)
1740+
struct gc_generation_stats *stats)
17191741
{
1720-
GC_STAT_ADD(2, collections, 1);
17211742
GCState *gcstate = &tstate->interp->gc;
17221743
validate_spaces(gcstate);
17231744
PyGC_Head *young = &gcstate->young.head;
@@ -1749,7 +1770,7 @@ static void
17491770
gc_collect_region(PyThreadState *tstate,
17501771
PyGC_Head *from,
17511772
PyGC_Head *to,
1752-
struct gc_collection_stats *stats)
1773+
struct gc_generation_stats *stats)
17531774
{
17541775
PyGC_Head unreachable; /* non-problematic unreachable trash */
17551776
PyGC_Head finalizers; /* objects with, & reachable from, __del__ */
@@ -1842,7 +1863,7 @@ gc_collect_region(PyThreadState *tstate,
18421863
*/
18431864
static void
18441865
do_gc_callback(GCState *gcstate, const char *phase,
1845-
int generation, struct gc_collection_stats *stats)
1866+
int generation, struct gc_generation_stats *stats)
18461867
{
18471868
assert(!PyErr_Occurred());
18481869

@@ -1890,7 +1911,7 @@ do_gc_callback(GCState *gcstate, const char *phase,
18901911

18911912
static void
18921913
invoke_gc_callback(GCState *gcstate, const char *phase,
1893-
int generation, struct gc_collection_stats *stats)
1914+
int generation, struct gc_generation_stats *stats)
18941915
{
18951916
if (gcstate->callbacks == NULL) {
18961917
return;
@@ -2082,7 +2103,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason)
20822103
}
20832104
gcstate->frame = tstate->current_frame;
20842105

2085-
struct gc_collection_stats stats = { 0 };
2106+
struct gc_generation_stats stats = { 0 };
20862107
if (reason != _Py_GC_REASON_SHUTDOWN) {
20872108
invoke_gc_callback(gcstate, "start", generation, &stats);
20882109
}
@@ -2093,8 +2114,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason)
20932114
if (PyDTrace_GC_START_ENABLED()) {
20942115
PyDTrace_GC_START(generation);
20952116
}
2096-
PyTime_t start, stop;
2097-
(void)PyTime_PerfCounterRaw(&start);
2117+
(void)PyTime_PerfCounterRaw(&stats.ts);
20982118
PyObject *exc = _PyErr_GetRaisedException(tstate);
20992119
switch(generation) {
21002120
case 0:
@@ -2109,8 +2129,9 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason)
21092129
default:
21102130
Py_UNREACHABLE();
21112131
}
2132+
PyTime_t stop;
21122133
(void)PyTime_PerfCounterRaw(&stop);
2113-
stats.duration = PyTime_AsSecondsDouble(stop - start);
2134+
stats.duration = PyTime_AsSecondsDouble(stop - stats.ts);
21142135
add_stats(gcstate, generation, &stats);
21152136
if (PyDTrace_GC_DONE_ENABLED()) {
21162137
PyDTrace_GC_DONE(stats.uncollectable + stats.collected);

0 commit comments

Comments
 (0)