From 1964b08394e01a5b6881013c0f34ee20073cc328 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Fri, 4 Sep 2020 12:01:52 -0700 Subject: [PATCH] HPA: Add stats for the hpa_shard. --- include/jemalloc/internal/arena_externs.h | 2 +- include/jemalloc/internal/ctl.h | 1 + include/jemalloc/internal/hpa.h | 7 ++ include/jemalloc/internal/mutex_prof.h | 4 +- include/jemalloc/internal/pa.h | 2 +- include/jemalloc/internal/psset.h | 7 ++ include/jemalloc/internal/stats.h | 3 +- src/arena.c | 4 +- src/ctl.c | 87 +++++++++++++++++- src/hpa.c | 1 - src/pa_extra.c | 22 ++++- src/stats.c | 106 ++++++++++++++++++++-- 12 files changed, 230 insertions(+), 16 deletions(-) diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index 9d4da31b..c8e1e38d 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -28,7 +28,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, bin_stats_data_t *bstats, arena_stats_large_t *lstats, - pac_estats_t *estats); + pac_estats_t *estats, hpa_shard_stats_t *hpastats); void arena_handle_new_dirty_pages(tsdn_t *tsdn, arena_t *arena); #ifdef JEMALLOC_JET size_t arena_slab_regind(edata_t *slab, szind_t binind, const void *ptr); diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index fbc432bf..305d3655 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -45,6 +45,7 @@ typedef struct ctl_arena_stats_s { bin_stats_data_t bstats[SC_NBINS]; arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; pac_estats_t estats[SC_NPSIZES]; + hpa_shard_stats_t hpastats; } ctl_arena_stats_t; typedef struct ctl_stats_s { diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h index 3decbf17..3fe9fc48 100644 --- a/include/jemalloc/internal/hpa.h +++ b/include/jemalloc/internal/hpa.h @@ -29,6 +29,13 @@ struct hpa_s { geom_grow_t geom_grow; }; +/* Used only by CTL; not actually stored here (i.e., all derived). */ +typedef struct hpa_shard_stats_s hpa_shard_stats_t; +struct hpa_shard_stats_s { + psset_bin_stats_t psset_full_slab_stats; + psset_bin_stats_t psset_slab_stats[PSSET_NPSIZES]; +}; + typedef struct hpa_shard_s hpa_shard_t; struct hpa_shard_s { /* diff --git a/include/jemalloc/internal/mutex_prof.h b/include/jemalloc/internal/mutex_prof.h index 91ab4114..970f469b 100644 --- a/include/jemalloc/internal/mutex_prof.h +++ b/include/jemalloc/internal/mutex_prof.h @@ -31,7 +31,9 @@ typedef enum { OP(decay_dirty) \ OP(decay_muzzy) \ OP(base) \ - OP(tcache_list) + OP(tcache_list) \ + OP(hpa_shard) \ + OP(hpa_shard_grow) typedef enum { #define OP(mtx) arena_prof_mutex_##mtx, diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 473d682b..d138f2f0 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -192,7 +192,7 @@ void pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out, - size_t *resident); + hpa_shard_stats_t *hpa_stats_out, size_t *resident); /* * Reads the PA-owned mutex stats into the output stats array, at the diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 72ff240e..7bba3cbc 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -31,6 +31,13 @@ struct psset_bin_stats_s { size_t ninactive; }; +static inline void +psset_bin_stats_accum(psset_bin_stats_t *dst, psset_bin_stats_t *src) { + dst->npageslabs += src->npageslabs; + dst->nactive += src->nactive; + dst->ninactive += src->ninactive; +} + typedef struct psset_s psset_t; struct psset_s { /* diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index 93bde22e..727f7dcb 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -11,7 +11,8 @@ OPTION('b', bins, true, false) \ OPTION('l', large, true, false) \ OPTION('x', mutex, true, false) \ - OPTION('e', extents, true, false) + OPTION('e', extents, true, false) \ + OPTION('h', hpa, config_stats, false) enum { #define OPTION(o, v, d, s) stats_print_option_num_##v, diff --git a/src/arena.c b/src/arena.c index 5fb5843b..dc58a287 100644 --- a/src/arena.c +++ b/src/arena.c @@ -81,7 +81,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, bin_stats_data_t *bstats, arena_stats_large_t *lstats, - pac_estats_t *estats) { + pac_estats_t *estats, hpa_shard_stats_t *hpastats) { cassert(config_stats); arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, @@ -139,7 +139,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, } pa_shard_stats_merge(tsdn, &arena->pa_shard, &astats->pa_shard_stats, - estats, &astats->resident); + estats, hpastats, &astats->resident); LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); diff --git a/src/ctl.c b/src/ctl.c index 9b8ab752..b4e65172 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -216,6 +216,13 @@ CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) INDEX_PROTO(stats_arenas_i_extents_j) +CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs) +CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive) +CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive) +INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j) +CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_npageslabs) +CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_nactive) +CTL_PROTO(stats_arenas_i_hpa_shard_full_slabs_ninactive) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_uptime) CTL_PROTO(stats_arenas_i_dss) @@ -584,6 +591,41 @@ MUTEX_PROF_ARENA_MUTEXES #undef OP }; +static const ctl_named_node_t stats_arenas_i_hpa_shard_full_slabs_node[] = { + {NAME("npageslabs"), + CTL(stats_arenas_i_hpa_shard_full_slabs_npageslabs)}, + {NAME("nactive"), + CTL(stats_arenas_i_hpa_shard_full_slabs_nactive)}, + {NAME("ninactive"), + CTL(stats_arenas_i_hpa_shard_full_slabs_ninactive)} +}; + +static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = { + {NAME("npageslabs"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs)}, + {NAME("nactive"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive)}, + {NAME("ninactive"), + CTL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive)} +}; + +static const ctl_named_node_t super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = { + {NAME(""), + CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)} +}; + +static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] = +{ + {INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)} +}; + +static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = { + {NAME("full_slabs"), CHILD(named, + stats_arenas_i_hpa_shard_full_slabs)}, + {NAME("nonfull_slabs"), CHILD(indexed, + stats_arenas_i_hpa_shard_nonfull_slabs)} +}; + static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, {NAME("uptime"), CTL(stats_arenas_i_uptime)}, @@ -613,7 +655,8 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, - {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} + {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)}, + {NAME("hpa_shard"), CHILD(named, stats_arenas_i_hpa_shard)} }; static const ctl_named_node_t super_stats_arenas_i_node[] = { {NAME(""), CHILD(named, stats_arenas_i)} @@ -844,6 +887,8 @@ ctl_arena_clear(ctl_arena_t *ctl_arena) { sizeof(arena_stats_large_t)); memset(ctl_arena->astats->estats, 0, SC_NPSIZES * sizeof(pac_estats_t)); + memset(&ctl_arena->astats->hpastats, 0, + sizeof(hpa_shard_stats_t)); } } @@ -857,7 +902,8 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, &ctl_arena->pdirty, &ctl_arena->pmuzzy, &ctl_arena->astats->astats, ctl_arena->astats->bstats, - ctl_arena->astats->lstats, ctl_arena->astats->estats); + ctl_arena->astats->lstats, ctl_arena->astats->estats, + &ctl_arena->astats->hpastats); for (i = 0; i < SC_NBINS; i++) { bin_stats_t *bstats = @@ -1033,6 +1079,16 @@ MUTEX_PROF_ARENA_MUTEXES sdstats->estats[i].retained_bytes += astats->estats[i].retained_bytes; } + + /* Merge HPA stats. */ + psset_bin_stats_accum(&sdstats->hpastats.psset_full_slab_stats, + &astats->hpastats.psset_full_slab_stats); + for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { + psset_bin_stats_accum( + &sdstats->hpastats.psset_slab_stats[i], + &astats->hpastats.psset_slab_stats[i]); + } + } } @@ -3256,6 +3312,33 @@ stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, return super_stats_arenas_i_extents_j_node; } +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_npageslabs, + arenas_i(mib[2])->astats->hpastats.psset_full_slab_stats.npageslabs, + size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_nactive, + arenas_i(mib[2])->astats->hpastats.psset_full_slab_stats.nactive, size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_full_slabs_ninactive, + arenas_i(mib[2])->astats->hpastats.psset_full_slab_stats.ninactive, size_t); + +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs, + arenas_i(mib[2])->astats->hpastats.psset_slab_stats[mib[5]].npageslabs, + size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive, + arenas_i(mib[2])->astats->hpastats.psset_slab_stats[mib[5]].nactive, + size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nonfull_slabs_j_ninactive, + arenas_i(mib[2])->astats->hpastats.psset_slab_stats[mib[5]].ninactive, + size_t); + +static const ctl_named_node_t * +stats_arenas_i_hpa_shard_nonfull_slabs_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j >= PSSET_NPSIZES) { + return NULL; + } + return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node; +} + static bool ctl_arenas_i_verify(size_t i) { size_t a = arenas_i2a_impl(i, true, true); diff --git a/src/hpa.c b/src/hpa.c index 597261d4..08992bda 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -258,7 +258,6 @@ hpa_from_pai(pai_t *self) { static edata_t * hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero) { - assert((size & PAGE_MASK) == 0); hpa_shard_t *shard = hpa_from_pai(self); /* We don't handle alignment or zeroing for now. */ diff --git a/src/pa_extra.c b/src/pa_extra.c index 402603ea..db236ad8 100644 --- a/src/pa_extra.c +++ b/src/pa_extra.c @@ -76,7 +76,7 @@ pa_shard_basic_stats_merge(pa_shard_t *shard, size_t *nactive, size_t *ndirty, void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out, - size_t *resident) { + hpa_shard_stats_t *hpa_stats_out, size_t *resident) { cassert(config_stats); pa_shard_stats_out->pac_stats.retained += @@ -138,6 +138,18 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, estats_out[i].muzzy_bytes = muzzy_bytes; estats_out[i].retained_bytes = retained_bytes; } + + if (shard->ever_used_hpa) { + malloc_mutex_lock(tsdn, &shard->hpa_shard.mtx); + psset_bin_stats_accum(&hpa_stats_out->psset_full_slab_stats, + &shard->hpa_shard.psset.full_slab_stats); + for (pszind_t i = 0; i < PSSET_NPSIZES; i++) { + psset_bin_stats_accum( + &hpa_stats_out->psset_slab_stats[i], + &shard->hpa_shard.psset.slab_stats[i]); + } + malloc_mutex_unlock(tsdn, &shard->hpa_shard.mtx); + } } static void @@ -163,4 +175,12 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard, &shard->pac.decay_dirty.mtx, arena_prof_mutex_decay_dirty); pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy); + + if (shard->ever_used_hpa) { + pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, + &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard); + pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, + &shard->hpa_shard.grow_mtx, + arena_prof_mutex_hpa_shard_grow); + } } diff --git a/src/stats.c b/src/stats.c index 7cbf2048..f03e5e44 100644 --- a/src/stats.c +++ b/src/stats.c @@ -43,6 +43,16 @@ const char *arena_mutex_names[mutex_prof_num_arena_mutexes] = { xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ } while (0) +#define CTL_M2_M5_GET(n, i, j, v, t) do { \ + size_t mib[CTL_MAX_DEPTH]; \ + size_t miblen = sizeof(mib) / sizeof(size_t); \ + size_t sz = sizeof(t); \ + xmallctlnametomib(n, mib, &miblen); \ + mib[2] = (i); \ + mib[5] = (j); \ + xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ +} while (0) + /******************************************************************************/ /* Data. */ @@ -650,6 +660,87 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { } } +static void +stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i) { + emitter_row_t header_row; + emitter_row_init(&header_row); + emitter_row_t row; + emitter_row_init(&row); + + size_t npageslabs; + size_t nactive; + size_t ninactive; + + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs", + i, &npageslabs, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive", + i, &nactive, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ninactive", + i, &ninactive, size_t); + + emitter_table_printf(emitter, + "HPA shard stats:\n" + " In full slabs:\n" + " npageslabs: %zu\n" + " nactive: %zu\n" + " ninactive: %zu\n", + npageslabs, nactive, ninactive); + emitter_json_object_kv_begin(emitter, "hpa_shard"); + emitter_json_object_kv_begin(emitter, "full_slabs"); + emitter_json_kv(emitter, "npageslabs", emitter_type_size, &npageslabs); + emitter_json_kv(emitter, "nactive", emitter_type_size, &nactive); + emitter_json_kv(emitter, "ninactive", emitter_type_size, &ninactive); + emitter_json_object_end(emitter); /* End "full_slabs" */ + + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, npageslabs, NULL, right, 13, size) + COL_HDR(row, nactive, NULL, right, 13, size) + COL_HDR(row, ninactive, NULL, right, 13, size) + + emitter_table_row(emitter, &header_row); + emitter_json_array_kv_begin(emitter, "nonfull_slabs"); + bool in_gap = false; + for (pszind_t j = 0; j < PSSET_NPSIZES; j++) { + CTL_M2_M5_GET( + "stats.arenas.0.hpa_shard.nonfull_slabs.0.npageslabs", + i, j, &npageslabs, size_t); + CTL_M2_M5_GET( + "stats.arenas.0.hpa_shard.nonfull_slabs.0.nactive", + i, j, &nactive, size_t); + CTL_M2_M5_GET( + "stats.arenas.0.hpa_shard.nonfull_slabs.0.ninactive", + i, j, &ninactive, size_t); + + bool in_gap_prev = in_gap; + in_gap = (npageslabs == 0); + if (in_gap_prev && !in_gap) { + emitter_table_printf(emitter, + " ---\n"); + } + + col_size.size_val = sz_pind2sz(j); + col_ind.size_val = j; + col_npageslabs.size_val = npageslabs; + col_nactive.size_val = nactive; + col_ninactive.size_val = ninactive; + if (!in_gap) { + emitter_table_row(emitter, &row); + } + + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "npageslabs", emitter_type_size, + &npageslabs); + emitter_json_kv(emitter, "nactive", emitter_type_size, + &nactive); + emitter_json_kv(emitter, "ninactive", emitter_type_size, + &ninactive); + emitter_json_object_end(emitter); + } + emitter_json_array_end(emitter); /* End "nonfull_slabs" */ + emitter_json_object_end(emitter); /* End "hpa_shard" */ +} + static void stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { emitter_row_t row; @@ -677,7 +768,7 @@ stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptim static void stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, - bool mutex, bool extents) { + bool mutex, bool extents, bool hpa) { unsigned nthreads; const char *dss; ssize_t dirty_decay_ms, muzzy_decay_ms; @@ -997,6 +1088,9 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, if (extents) { stats_arena_extents_print(emitter, i); } + if (hpa) { + stats_arena_hpa_shard_print(emitter, i); + } } static void @@ -1272,7 +1366,7 @@ stats_general_print(emitter_t *emitter) { static void stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, - bool unmerged, bool bins, bool large, bool mutex, bool extents) { + bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) { /* * These should be deleted. We keep them around for a while, to aid in * the transition to the emitter code. @@ -1405,7 +1499,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, emitter_table_printf(emitter, "Merged arenas stats:\n"); emitter_json_object_kv_begin(emitter, "merged"); stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins, - large, mutex, extents); + large, mutex, extents, hpa); emitter_json_object_end(emitter); /* Close "merged". */ } @@ -1416,7 +1510,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, "Destroyed arenas stats:\n"); emitter_json_object_kv_begin(emitter, "destroyed"); stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, - bins, large, mutex, extents); + bins, large, mutex, extents, hpa); emitter_json_object_end(emitter); /* Close "destroyed". */ } @@ -1432,7 +1526,7 @@ stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, emitter_table_printf(emitter, "arenas[%s]:\n", arena_ind_str); stats_arena_print(emitter, i, bins, - large, mutex, extents); + large, mutex, extents, hpa); /* Close "". */ emitter_json_object_end(emitter); } @@ -1497,7 +1591,7 @@ stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { } if (config_stats) { stats_print_helper(&emitter, merged, destroyed, unmerged, - bins, large, mutex, extents); + bins, large, mutex, extents, hpa); } emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */