From 89dc90aafaa1c322b48fdfb52393b090034a4862 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 29 Jul 2023 14:23:37 +0100 Subject: [PATCH 1/5] Add GC stats to Py_STATS --- Include/internal/pycore_code.h | 2 ++ Include/pystats.h | 9 +++++++++ Modules/gcmodule.c | 18 ++++++++++++++++++ Python/specialize.c | 14 +++++++++++++- Tools/scripts/summarize_stats.py | 26 ++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index bcdf8645c8557c..5d3262222c7b80 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -272,6 +272,7 @@ extern int _PyStaticCode_Init(PyCodeObject *co); #define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \ do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0) +#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0) // Export for '_opcode' shared extension PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); @@ -285,6 +286,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define OBJECT_STAT_INC_COND(name, cond) ((void)0) #define EVAL_CALL_STAT_INC(name) ((void)0) #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) ((void)0) +#define GC_STAT_ADD(gen, name, n) ((void)0) #endif // !Py_STATS // Utility functions for reading/writing 32/64-bit values in the inline caches. diff --git a/Include/pystats.h b/Include/pystats.h index 54c9b8d8b3538f..e24aef5fe8072b 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -74,12 +74,21 @@ typedef struct _object_stats { uint64_t optimization_traces_created; uint64_t optimization_traces_executed; uint64_t optimization_uops_executed; + /* Temporary value used during GC */ + uint64_t object_visits; } ObjectStats; +typedef struct _gc_stats { + uint64_t collections; + uint64_t object_visits; + uint64_t objects_collected; +} GCStats; + typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; ObjectStats object_stats; + GCStats *gc_stats; } PyStats; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 246c0a9e160aa9..35a35091bf4511 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -460,6 +460,7 @@ update_refs(PyGC_Head *containers) static int visit_decref(PyObject *op, void *parent) { + OBJECT_STAT_INC(object_visits); _PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op)); if (_PyObject_IS_GC(op)) { @@ -498,6 +499,7 @@ subtract_refs(PyGC_Head *containers) static int visit_reachable(PyObject *op, PyGC_Head *reachable) { + OBJECT_STAT_INC(object_visits); if (!_PyObject_IS_GC(op)) { return 0; } @@ -725,6 +727,7 @@ clear_unreachable_mask(PyGC_Head *unreachable) static int visit_move(PyObject *op, PyGC_Head *tolist) { + OBJECT_STAT_INC(object_visits); if (_PyObject_IS_GC(op)) { PyGC_Head *gc = AS_GC(op); if (gc_is_collecting(gc)) { @@ -1195,6 +1198,12 @@ gc_collect_main(PyThreadState *tstate, int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, int nofail) { + GC_STAT_ADD(generation, collections, 1); +#ifdef Py_STATS + if (_py_stats) { + _py_stats->object_stats.object_visits = 0; + } +#endif int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ @@ -1351,6 +1360,15 @@ gc_collect_main(PyThreadState *tstate, int generation, stats->collected += m; stats->uncollectable += n; + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_py_stats) { + GC_STAT_ADD(generation, object_visits, + _py_stats->object_stats.object_visits); + _py_stats->object_stats.object_visits = 0; + } +#endif + if (PyDTrace_GC_DONE_ENABLED()) { PyDTrace_GC_DONE(n + m); } diff --git a/Python/specialize.c b/Python/specialize.c index 1669ce17fc804e..f60e7a80649775 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -18,7 +18,8 @@ */ #ifdef Py_STATS -PyStats _py_stats_struct = { 0 }; +GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 }; +PyStats _py_stats_struct = { .gc_stats = &_py_gc_stats[0] }; PyStats *_py_stats = NULL; #define ADD_STAT_TO_DICT(res, field) \ @@ -202,11 +203,22 @@ print_object_stats(FILE *out, ObjectStats *stats) fprintf(out, "Optimization uops executed: %" PRIu64 "\n", stats->optimization_uops_executed); } +static void +print_gc_stats(FILE *out, GCStats *stats) +{ + for (int i = 0; i < NUM_GENERATIONS; i++) { + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats->collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats->object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats->objects_collected); + } +} + static void print_stats(FILE *out, PyStats *stats) { print_spec_stats(out, stats->opcode_stats); print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); + print_gc_stats(out, stats->gc_stats); } void diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 9c881897c2de1d..76f856905e78b7 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -494,17 +494,41 @@ def calculate_object_stats(stats): rows.append((label, value, ratio)) return rows +def calculate_gc_stats(stats): + gc_stats = [] + for key, value in stats.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return [ + (i, gen["collections"], gen["objects collected"], gen["object visits"]) + for (i, gen) in enumerate(gc_stats) + ] + def emit_object_stats(stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): rows = calculate_object_stats(stats) emit_table(("", "Count:", "Ratio:"), rows) +def emit_gc_stats(stats): + with Section("GC stats", summary="allocations, frees and dict materializatons"): + rows = calculate_gc_stats(stats) + emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) + def emit_comparative_object_stats(base_stats, head_stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): base_rows = calculate_object_stats(base_stats) head_rows = calculate_object_stats(head_stats) emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), join_rows(base_rows, head_rows)) +def emit_comparative_gc_stats(base_stats, head_stats): + pass # To do... + def get_total(opcode_stats): total = 0 for opcode_stat in opcode_stats: @@ -574,6 +598,7 @@ def output_single_stats(stats): emit_specialization_overview(opcode_stats, total) emit_call_stats(stats) emit_object_stats(stats) + emit_gc_stats(stats) with Section("Meta stats", summary="Meta statistics"): emit_table(("", "Count:"), [('Number of data files', stats['__nfiles__'])]) @@ -596,6 +621,7 @@ def output_comparative_stats(base_stats, head_stats): ) emit_comparative_call_stats(base_stats, head_stats) emit_comparative_object_stats(base_stats, head_stats) + emit_comparative_gc_stats(base_stats, head_stats) def output_stats(inputs, json_output=None): if len(inputs) == 1: From 90a79be8e8234d02c7e43d46880f470aecacf1ee Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 29 Jul 2023 19:02:17 +0100 Subject: [PATCH 2/5] Clear stats correctly. --- Python/specialize.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/specialize.c b/Python/specialize.c index f60e7a80649775..a4f721b946723b 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -224,7 +224,11 @@ print_stats(FILE *out, PyStats *stats) { void _Py_StatsClear(void) { + for (int i = 0; i < NUM_GENERATIONS; i++) { + _py_gc_stats[i] = (GCStats) { 0 }; + } _py_stats_struct = (PyStats) { 0 }; + _py_stats_struct.gc_stats = _py_gc_stats; } void From 8bc0926a995447cb4ae9c9f2b7182c818bbe08d4 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sun, 30 Jul 2023 18:20:43 +0100 Subject: [PATCH 3/5] Print out GC stats for generations correctly. --- Python/specialize.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/specialize.c b/Python/specialize.c index a4f721b946723b..de329ef1195cbf 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -207,9 +207,9 @@ static void print_gc_stats(FILE *out, GCStats *stats) { for (int i = 0; i < NUM_GENERATIONS; i++) { - fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats->collections); - fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats->object_visits); - fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats->objects_collected); + fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); + fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); + fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); } } From 0d3bae7e4e60a9b7bb3b0d9b3125332363b513b3 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 31 Jul 2023 02:33:49 +0100 Subject: [PATCH 4/5] Add comparative GC stats --- Tools/scripts/summarize_stats.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 76f856905e78b7..1e175ed449d9a0 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -515,19 +515,27 @@ def emit_object_stats(stats): rows = calculate_object_stats(stats) emit_table(("", "Count:", "Ratio:"), rows) -def emit_gc_stats(stats): - with Section("GC stats", summary="allocations, frees and dict materializatons"): - rows = calculate_gc_stats(stats) - emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) - def emit_comparative_object_stats(base_stats, head_stats): with Section("Object stats", summary="allocations, frees and dict materializatons"): base_rows = calculate_object_stats(base_stats) head_rows = calculate_object_stats(head_stats) emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), join_rows(base_rows, head_rows)) +def emit_gc_stats(stats): + with Section("GC stats", summary"GC collections and effectiveness"): + rows = calculate_gc_stats(stats) + emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) + def emit_comparative_gc_stats(base_stats, head_stats): - pass # To do... + with Section("GC stats", summary"GC collections and effectiveness"): + base_rows = calculate_gc_stats(base_stats) + head_rows = calculate_gc_stats(head_stats) + emit_table( + ("Generation:", + "Base collections:", "Head collections:", + "Base objects collected:", "Head objects collected:", + "Base object visits:", "Head object visits:"), + join_rows(base_rows, head_rows)) def get_total(opcode_stats): total = 0 From 8b7489fa4cf09d90e32b3e850906a195ac4721cc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 31 Jul 2023 06:01:09 +0100 Subject: [PATCH 5/5] Fix typo --- Tools/scripts/summarize_stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 1e175ed449d9a0..f798b2f772d08a 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -522,12 +522,12 @@ def emit_comparative_object_stats(base_stats, head_stats): emit_table(("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), join_rows(base_rows, head_rows)) def emit_gc_stats(stats): - with Section("GC stats", summary"GC collections and effectiveness"): + with Section("GC stats", summary="GC collections and effectiveness"): rows = calculate_gc_stats(stats) emit_table(("Generation:", "Collections:", "Objects collected:", "Object visits:"), rows) def emit_comparative_gc_stats(base_stats, head_stats): - with Section("GC stats", summary"GC collections and effectiveness"): + with Section("GC stats", summary="GC collections and effectiveness"): base_rows = calculate_gc_stats(base_stats) head_rows = calculate_gc_stats(head_stats) emit_table( pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy