From bee1cf46a8a2e45daeb44cadc6e1fb021e3f7fdd Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Wed, 15 Jul 2026 23:06:25 -0700 Subject: [PATCH 1/9] Fix duplicate opt.stats_print / opt.stats_print_opts in malloc_stats_print --- src/stats.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/stats.c b/src/stats.c index d4b0f4e6..d88b8acb 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1840,8 +1840,6 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("prof_leak_error") OPT_WRITE_BOOL("stats_print") OPT_WRITE_CHAR_P("stats_print_opts") - OPT_WRITE_BOOL("stats_print") - OPT_WRITE_CHAR_P("stats_print_opts") OPT_WRITE_INT64("stats_interval") OPT_WRITE_CHAR_P("stats_interval_opts") OPT_WRITE_CHAR_P("zero_realloc") From ab2520fe948802626796b8007b45a85813a45186 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Thu, 16 Jul 2026 07:06:27 -0700 Subject: [PATCH 2/9] Refactor stats.c: separate gathering from emission; read top-down src/stats.c fused mallctl reads with dual-format (text/JSON) emission inside a few very large, bottom-up functions, behind dense macros, with the public entry point at the very bottom. Rework it to be output-preserving but much easier to read and extend: - Separate gathering from emission where it is worth it: most sections become a small gather (stats_gather_*, the mallctl reads into a typed struct) plus an emit that renders it, with the plumbing macros (CTL_*, COL*) and gather structs moved to internal header (stats_internal.h). This is a balance, not a rule: the per-size-class tables stay O(1) streaming (per-row gather+emit, no large stack buffers, avoiding heap buffering), the mutex rows keep reading straight into emitter columns, and the already-readable config/opt probes are left as they are. - Read like the output: open with a table-of-contents comment and the public stats_print(), then the section functions top-down, keeping the interval/boot/fork plumbing at the bottom. The ToC names the function that prints each section, and its JSON sub-path. - Keep one emission engine: the emitter remains the single dual-format renderer, so each section has one rendering path rather than parallel renderers (the two formats are not field-identical). Move some table-only concerns into it -- gap-collapsing ("---") and a dual columnar row op (emitter_row) -- so section code carries less table-format detail. Output-preserving: text is byte-identical and JSON keys/values are unchanged. --- include/jemalloc/internal/emitter.h | 91 + include/jemalloc/internal/stats_internal.h | 340 +++ src/stats.c | 2218 ++++++++++---------- test/unit/emitter.c | 130 +- 4 files changed, 1718 insertions(+), 1061 deletions(-) create mode 100644 include/jemalloc/internal/stats_internal.h diff --git a/include/jemalloc/internal/emitter.h b/include/jemalloc/internal/emitter.h index a4073e6a..9e645106 100644 --- a/include/jemalloc/internal/emitter.h +++ b/include/jemalloc/internal/emitter.h @@ -58,6 +58,11 @@ struct emitter_col_s { ssize_t ssize_val; const char *str_val; }; + /* + * Optional JSON key. When set, emitter_row() emits this column as a + * "json_key": value pair in JSON mode; columns left NULL are table-only. + */ + const char *json_key; /* Filled in by initialization. */ ql_elm(emitter_col_t) link; @@ -79,6 +84,8 @@ struct emitter_s { bool item_at_depth; /* True if we emitted a key and will emit corresponding value next. */ bool emitted_key; + /* Gap-collapsing state for the emitter_table_sparse_row() bracket. */ + bool table_sparse_in_gap; }; static inline bool @@ -270,6 +277,7 @@ emitter_init(emitter_t *emitter, emitter_output_t emitter_output, emitter->item_at_depth = false; emitter->emitted_key = false; emitter->nesting_depth = 0; + emitter->table_sparse_in_gap = false; } /******************************************************************************/ @@ -447,6 +455,46 @@ static inline void emitter_col_init(emitter_col_t *col, emitter_row_t *row) { ql_elm_new(col, link); ql_tail_insert(&row->cols, col, link); + col->json_key = NULL; +} + +/* + * Sparse table rows. Some tables have one row per size class but most rows + * are all-zero; in table output those runs are collapsed to a single "---" + * separator. Bracket such a table with emitter_table_sparse_begin() / + * _end() and emit each row with emitter_table_sparse_row(), passing whether + * that row is a gap (all-zero). The emitter prints the separator on a + * gap->non-gap transition and at a trailing gap, and suppresses the gap rows + * themselves; callers no longer track gap state. Table output only -- a + * no-op in JSON, where callers emit every row's object separately. + */ +static inline void +emitter_table_sparse_begin(emitter_t *emitter) { + emitter->table_sparse_in_gap = false; +} + +static inline void +emitter_table_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) { + if (emitter->output != emitter_output_table) { + return; + } + bool prev_in_gap = emitter->table_sparse_in_gap; + emitter->table_sparse_in_gap = is_gap; + if (prev_in_gap && !is_gap) { + emitter_printf(emitter, " ---\n"); + } + if (!is_gap) { + emitter_table_row(emitter, row); + } +} + +static inline void +emitter_table_sparse_end(emitter_t *emitter) { + if (emitter->output == emitter_output_table + && emitter->table_sparse_in_gap) { + emitter_printf(emitter, " ---\n"); + } + emitter->table_sparse_in_gap = false; } /******************************************************************************/ @@ -499,6 +547,49 @@ emitter_dict_end(emitter_t *emitter) { } } +/* Internal: emit a row's json-keyed columns as JSON kvs, in column order. */ +static inline void +emitter_row_json(emitter_t *emitter, emitter_row_t *row) { + emitter_col_t *col; + ql_foreach (col, &row->cols, link) { + if (col->json_key != NULL) { + emitter_json_kv(emitter, col->json_key, col->type, + (const void *)&col->bool_val); + } + } +} + +/* + * Emit a row of columns to whichever format is active. In table mode, prints + * the aligned table row (see emitter_table_row). In JSON mode, emits each + * column that has a non-NULL json_key as a "json_key": value pair, in column + * order (columns without a json_key are table-only). This lets a caller + * describe a row once and render it to either format. For JSON objects (one + * per row), wrap the call in emitter_json_object_begin()/_end(). + */ +static inline void +emitter_row(emitter_t *emitter, emitter_row_t *row) { + emitter_table_row(emitter, row); + if (emitter_outputs_json(emitter)) { + emitter_row_json(emitter, row); + } +} + +/* + * Sparse counterpart of emitter_row() for per-size-class tables: the table + * side collapses all-zero ("gap") rows (see emitter_table_sparse_row), while + * the JSON side still emits every row's json-keyed columns. Bracket the loop + * with emitter_table_sparse_begin()/_end(), and wrap each row's JSON in + * emitter_json_object_begin()/_end() when the rows form an array of objects. + */ +static inline void +emitter_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) { + emitter_table_sparse_row(emitter, row, is_gap); + if (emitter_outputs_json(emitter)) { + emitter_row_json(emitter, row); + } +} + static inline void emitter_begin(emitter_t *emitter) { if (emitter_outputs_json(emitter)) { diff --git a/include/jemalloc/internal/stats_internal.h b/include/jemalloc/internal/stats_internal.h new file mode 100644 index 00000000..0c3f7b2b --- /dev/null +++ b/include/jemalloc/internal/stats_internal.h @@ -0,0 +1,340 @@ +#ifndef JEMALLOC_INTERNAL_STATS_INTERNAL_H +#define JEMALLOC_INTERNAL_STATS_INTERNAL_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena.h" /* ARENA_NAME_LEN */ +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/emitter.h" +#include "jemalloc/internal/prof_stats.h" + +/* + * Internal implementation support for src/stats.c only -- do NOT include from + * other translation units. + * + * This header holds noisy local scaffolding for malloc_stats_print(): ctl + * gather plumbing, gather structs, emitter table-column helpers, and option + * list helpers. It is deliberately not a reusable stats-gather API. + */ + +#define CTL_GET(n, v, t) \ + do { \ + size_t sz = sizeof(t); \ + xmallctl(n, (void *)v, &sz, NULL, 0); \ + } while (0) + +#define CTL_LEAF_PREPARE(mib, miblen, name) \ + do { \ + assert(miblen < CTL_MAX_DEPTH); \ + size_t miblen_new = CTL_MAX_DEPTH; \ + xmallctlmibnametomib(mib, miblen, name, &miblen_new); \ + assert(miblen_new > miblen); \ + } while (0) + +#define CTL_LEAF(mib, miblen, leaf, v, t) \ + do { \ + assert(miblen < CTL_MAX_DEPTH); \ + size_t miblen_new = CTL_MAX_DEPTH; \ + size_t sz = sizeof(t); \ + xmallctlbymibname( \ + mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0); \ + assert(miblen_new == miblen + 1); \ + } while (0) + +#define CTL_MIB_GET(n, i, v, t, ind) \ + do { \ + size_t mib[CTL_MAX_DEPTH]; \ + size_t miblen = sizeof(mib) / sizeof(size_t); \ + size_t sz = sizeof(t); \ + xmallctlnametomib(n, mib, &miblen); \ + mib[(ind)] = (i); \ + xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ + } while (0) + +#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1) +#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2) + +/******************************************************************************/ +/* + * Per-section gather structs. The stats_gather_*() functions that fill them + * are static in src/stats.c. + */ + +/* One arena's memory-accounting stats (stats.arenas..*). */ +typedef struct stats_arena_mem_s { + size_t mapped; + size_t retained; + size_t pinned; + size_t base; + size_t internal; + size_t metadata_edata; + size_t metadata_rtree; + size_t metadata_thp; + size_t tcache_bytes; + size_t tcache_stashed_bytes; + size_t resident; + size_t abandoned_vm; + size_t extent_avail; + size_t page; /* arenas.page, for the active-bytes conversion */ +} stats_arena_mem_t; + +/* One arena extents row (stats.arenas..extents..*). */ +typedef struct stats_arena_extent_s { + size_t ndirty; + size_t nmuzzy; + size_t nretained; + size_t npinned; + size_t ntotal; + size_t dirty_bytes; + size_t muzzy_bytes; + size_t retained_bytes; + size_t pinned_bytes; + size_t total_bytes; +} stats_arena_extent_t; + +/* One arena's "basics" (stats.arenas..{nthreads,uptime,dss}). */ +typedef struct stats_arena_basics_s { + unsigned nthreads; + uint64_t uptime; + const char *dss; + char name[ARENA_NAME_LEN]; + bool has_name; /* false for the merged/destroyed pseudo-arenas */ +} stats_arena_basics_t; + +/* One arena's decay / page stats (stats.arenas..*). */ +typedef struct stats_arena_decay_s { + ssize_t dirty_decay_ms; + ssize_t muzzy_decay_ms; + size_t pactive; + size_t pdirty; + size_t pmuzzy; + uint64_t dirty_npurge; + uint64_t dirty_nmadvise; + uint64_t dirty_purged; + uint64_t muzzy_npurge; + uint64_t muzzy_nmadvise; + uint64_t muzzy_purged; +} stats_arena_decay_t; + +/* One size class of arena allocation counts (stats.arenas..{small,large}.*). */ +typedef struct stats_arena_alloc_s { + size_t allocated; + uint64_t nmalloc; + uint64_t ndalloc; + uint64_t nrequests; + uint64_t nfills; + uint64_t nflushes; +} stats_arena_alloc_t; + +/* + * One arena bin row (stats.arenas..bins..* + arenas.bin..*, + * plus prof.stats.bins..{live,accum} when profiling stats are enabled). + */ +typedef struct stats_arena_bin_s { + uint64_t nslabs; + uint64_t nmalloc; + uint64_t ndalloc; + uint64_t nrequests; + uint64_t nfills; + uint64_t nflushes; + uint64_t nreslabs; + size_t reg_size; + size_t slab_size; + size_t curregs; + size_t curslabs; + size_t nonfull_slabs; + uint32_t nregs; + uint32_t nshards; + prof_stats_t prof_live; + prof_stats_t prof_accum; +} stats_arena_bin_t; + +/* + * One arena large-extent row (stats.arenas..lextents..* + + * arenas.lextent..*, plus prof.stats.lextents..{live,accum} when + * profiling stats are enabled). + */ +typedef struct stats_arena_lextent_s { + uint64_t nmalloc; + uint64_t ndalloc; + uint64_t nrequests; + size_t lextent_size; + size_t curlextents; + prof_stats_t prof_live; + prof_stats_t prof_accum; +} stats_arena_lextent_t; + +/* HPA shard small-extent cache (stats.arenas..hpa_sec_*). */ +typedef struct stats_arena_hpa_sec_s { + size_t sec_bytes; + size_t sec_hits; + size_t sec_misses; + size_t sec_dalloc_flush; + size_t sec_dalloc_noflush; + size_t sec_overfills; +} stats_arena_hpa_sec_t; + +/* PAC small-extent cache (stats.arenas..pac_sec_*). */ +typedef struct stats_arena_pac_sec_s { + size_t sec_bytes; + size_t sec_hits; + size_t sec_misses; + size_t sec_dalloc_flush; + size_t sec_dalloc_noflush; +} stats_arena_pac_sec_t; + +/* HPA shard counters (stats.arenas..hpa_shard.*). */ +typedef struct stats_arena_hpa_counters_s { + size_t npageslabs; + size_t nactive; + size_t ndirty; + size_t npageslabs_nonhuge; + size_t nactive_nonhuge; + size_t ndirty_nonhuge; + size_t nretained_nonhuge; + size_t npageslabs_huge; + size_t nactive_huge; + size_t ndirty_huge; + uint64_t npurge_passes; + uint64_t npurges; + uint64_t nhugifies; + uint64_t nhugify_failures; + uint64_t ndehugifies; +} stats_arena_hpa_counters_t; + +/* One HPA slab class (full/empty/nonfull) row. */ +typedef struct stats_arena_hpa_slab_s { + size_t npageslabs_huge; + size_t nactive_huge; + size_t ndirty_huge; + size_t npageslabs_nonhuge; + size_t nactive_nonhuge; + size_t ndirty_nonhuge; + size_t nretained_nonhuge; +} stats_arena_hpa_slab_t; + +/* Arena configuration / size-class scalars (arenas.*). */ +typedef struct stats_arena_config_s { + unsigned narenas; + ssize_t dirty_decay_ms; + ssize_t muzzy_decay_ms; + size_t quantum; + size_t page; + size_t hugepage; + bool have_tcache_max; + size_t tcache_max; + unsigned nbins; + unsigned nhbins; + unsigned nlextents; +} stats_arena_config_t; + +/* Per-size-class geometry (emitted in JSON only). */ +typedef struct stats_arena_bin_meta_s { + size_t size; + uint32_t nregs; + size_t slab_size; + uint32_t nshards; +} stats_arena_bin_meta_t; + +typedef struct stats_arena_lextent_meta_s { + size_t size; +} stats_arena_lextent_meta_t; + +/* Process-wide global stats (stats.* + stats.background_thread.*). */ +typedef struct stats_global_s { + size_t allocated; + size_t active; + size_t metadata; + size_t metadata_edata; + size_t metadata_rtree; + size_t metadata_thp; + size_t resident; + size_t mapped; + size_t retained; + size_t pinned; + size_t zero_reallocs; + size_t num_background_threads; + uint64_t background_thread_num_runs; + uint64_t background_thread_run_interval; +} stats_global_t; + +/******************************************************************************/ +/* + * Emitter table-column helper macros: declare an emitter_col_t (and, for + * COL_HDR, its header column) and wire it into a row. + */ + +#define COL_DECLARE(column_name) emitter_col_t col_##column_name; + +#define COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_init(&col_##column_name, &row_name); \ + col_##column_name.justify = emitter_justify_##left_or_right; \ + col_##column_name.width = col_width; \ + col_##column_name.type = emitter_type_##etype; + +#define COL(row_name, column_name, left_or_right, col_width, etype) \ + COL_DECLARE(column_name); \ + COL_INIT(row_name, column_name, left_or_right, col_width, etype) + +#define COL_HDR_DECLARE(column_name) \ + COL_DECLARE(column_name); \ + emitter_col_t header_##column_name; + +#define COL_HDR_INIT( \ + row_name, column_name, human, left_or_right, col_width, etype) \ + COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_init(&header_##column_name, &header_##row_name); \ + header_##column_name.justify = emitter_justify_##left_or_right; \ + header_##column_name.width = col_width; \ + header_##column_name.type = emitter_type_title; \ + header_##column_name.str_val = human ? human : #column_name; + +#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ + COL_HDR_DECLARE(column_name) \ + COL_HDR_INIT( \ + row_name, column_name, human, left_or_right, col_width, etype) + +/* + * stats_general_print() config/option list helpers. Unhygienic by design: + * they assume `emitter` and the standard scratch locals are in scope at the + * call site -- bv/bsz (bool), cpv/cpsz (const char *), uv/usz (unsigned), + * i64v/i64sz, u64v/u64sz, sv/ssz (size_t), ssv/sssz (ssize_t), plus bv2/ssv2 + * for the *_MUTABLE variants. (CONFIG_WRITE_BOOL uses CTL_GET above; the + * OPT_WRITE_* variants use je_mallctl directly.) + */ +#define CONFIG_WRITE_BOOL(name) \ + do { \ + CTL_GET("config." #name, &bv, bool); \ + emitter_kv( \ + emitter, #name, "config." #name, emitter_type_bool, &bv); \ + } while (0) + +#define OPT_WRITE(name, var, size, emitter_type) \ + if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \ + emitter_kv(emitter, name, "opt." name, emitter_type, &var); \ + } + +#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname) \ + if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0 \ + && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) { \ + emitter_kv_note(emitter, name, "opt." name, emitter_type, \ + &var1, altname, emitter_type, &var2); \ + } + +#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool) +#define OPT_WRITE_BOOL_MUTABLE(name, altname) \ + OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname) + +#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned) + +#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64) +#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64) + +#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size) +#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize) +#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ + OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname) + +#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string) + +#endif /* JEMALLOC_INTERNAL_STATS_INTERNAL_H */ diff --git a/src/stats.c b/src/stats.c index d88b8acb..c09b49d4 100644 --- a/src/stats.c +++ b/src/stats.c @@ -10,6 +10,7 @@ #include "jemalloc/internal/prof.h" #include "jemalloc/internal/prof_inlines.h" #include "jemalloc/internal/prof_stats.h" +#include "jemalloc/internal/stats_internal.h" #include "jemalloc/internal/tcache.h" static const char *const global_mutex_names[mutex_prof_num_global_mutexes] = { @@ -24,43 +25,6 @@ static const char *const arena_mutex_names[mutex_prof_num_arena_mutexes] = { #undef OP }; -#define CTL_GET(n, v, t) \ - do { \ - size_t sz = sizeof(t); \ - xmallctl(n, (void *)v, &sz, NULL, 0); \ - } while (0) - -#define CTL_LEAF_PREPARE(mib, miblen, name) \ - do { \ - assert(miblen < CTL_MAX_DEPTH); \ - size_t miblen_new = CTL_MAX_DEPTH; \ - xmallctlmibnametomib(mib, miblen, name, &miblen_new); \ - assert(miblen_new > miblen); \ - } while (0) - -#define CTL_LEAF(mib, miblen, leaf, v, t) \ - do { \ - assert(miblen < CTL_MAX_DEPTH); \ - size_t miblen_new = CTL_MAX_DEPTH; \ - size_t sz = sizeof(t); \ - xmallctlbymibname( \ - mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0); \ - assert(miblen_new == miblen + 1); \ - } while (0) - -#define CTL_MIB_GET(n, i, v, t, ind) \ - do { \ - size_t mib[CTL_MAX_DEPTH]; \ - size_t miblen = sizeof(mib) / sizeof(size_t); \ - size_t sz = sizeof(t); \ - xmallctlnametomib(n, mib, &miblen); \ - mib[(ind)] = (i); \ - xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ - } while (0) - -#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1) -#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2) - /******************************************************************************/ /* Data. */ @@ -74,6 +38,471 @@ static counter_accum_t stats_interval_accumulated; /* Per thread batch accum size for stats_interval. */ uint64_t stats_interval_accum_batch; +/* + * Forward declarations for the two top-level section helpers. The public + * stats_print() appears first (just below) and calls these; their definitions + * follow it, so the file reads top-down. + */ +static void stats_general_print(emitter_t *emitter, bool omit_size_class_meta); +static void stats_print_runtime_stats(emitter_t *emitter, bool merged, + bool destroyed, + bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa); + +/******************************************************************************/ +/* + * malloc_stats_print() output structure (produced by stats_print(), below); + * each line names the function that prints it, plus its JSON key where useful. + * Quoted names are literal keys below the "jemalloc" root unless shown with a + * leading dot. + * + * ___ Begin jemalloc statistics ___ + * { "jemalloc": + * general [g] -> stats_general_print + * version -> stats_general_version + * config -> stats_general_config + * system -> stats_general_system + * opt.* -> stats_general_opts + * prof -> stats_general_prof + * arenas metadata -> stats_general_arenas_print ("arenas") + * stats (config_stats) -> stats_print_runtime_stats + * global totals -> stats_print_globals ("stats") + * background_thread -> stats_print_globals + * mutexes [x] -> stats_global_mutexes_print + * per arena [m]/[d]/[a] -> stats_arena_print ("stats.arenas") + * basics -> stats_arena_basics_print + * decay -> stats_arena_decay_print + * alloc counts -> stats_arena_alloc_print (.small/.large) + * memory -> stats_arena_mem_print + * pac/sec -> stats_arena_pac_sec_print + * mutexes [x] -> stats_arena_mutexes_print (.mutexes) + * bins [b] -> stats_arena_bins_print (.bins[j]) + * lextents [l] -> stats_arena_lextents_print (.lextents[j]) + * extents [e] -> stats_arena_extents_print (.extents[j]) + * hpa shard [h] -> stats_arena_hpa_shard_print (.hpa_shard) + * } + * --- End jemalloc statistics --- + * + * Most refactored sections read values through mallctl into typed structs + * before rendering them through the emitter. This makes their coverage easier + * to audit, but does not guarantee parity between formats: format-specific + * emission remains, including JSON-only geometry and table-only derived rates + * and summary rows. + * + * Mallctl names identify data sources, not mechanical JSON paths. For example, + * stats_gather_arena_alloc reads "stats.arenas.0.small.allocated" (replacing + * the MIB arena index at runtime), while the emitter writes it below + * jemalloc["stats.arenas"][].small.allocated. In particular, "stats" + * and "stats.arenas" are sibling keys, not nested objects. + */ + +void +stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { + int err; + uint64_t epoch; + size_t u64sz; +#define OPTION(o, v, d, s) bool v = d; + STATS_PRINT_OPTIONS +#undef OPTION + + /* + * Refresh stats, in case mallctl() was called by the application. + * + * Check for OOM here, since refreshing the ctl cache can trigger + * allocation. In practice, none of the subsequent mallctl()-related + * calls in this function will cause OOM if this one succeeds. + * */ + epoch = 1; + u64sz = sizeof(uint64_t); + err = je_mallctl( + "epoch", (void *)&epoch, &u64sz, (void *)&epoch, sizeof(uint64_t)); + if (err != 0) { + if (err == EAGAIN) { + malloc_write( + ": Memory allocation failure in " + "mallctl(\"epoch\", ...)\n"); + return; + } + malloc_write( + ": Failure in mallctl(\"epoch\", " + "...)\n"); + abort(); + } + + if (opts != NULL) { + for (unsigned i = 0; opts[i] != '\0'; i++) { + switch (opts[i]) { +#define OPTION(o, v, d, s) \ + case o: \ + v = s; \ + break; + STATS_PRINT_OPTIONS +#undef OPTION + default:; + } + } + } + + emitter_t emitter; + emitter_init(&emitter, + json ? emitter_output_json_compact : emitter_output_table, write_cb, + cbopaque); + emitter_begin(&emitter); + emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); + emitter_json_object_kv_begin(&emitter, "jemalloc"); + + if (general) { + /* + * Private (non-public) omission: the per-size-class geometry + * arrays are shown only in JSON; the raw format never did. + */ + bool omit_size_class_meta = !json; + stats_general_print(&emitter, omit_size_class_meta); + } + if (config_stats) { + stats_print_runtime_stats(&emitter, merged, destroyed, unmerged, + bins, large, mutex, extents, hpa); + } + + emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ + emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); + emitter_end(&emitter); +} + +/******************************************************************************/ +/* Statistics gathering. */ +/* + * Read stats out of the allocator through mallctl into typed per-section + * structs. Keeping the mallctl reads together (and separate from emission) + * makes coverage auditable. They are static to src/stats.c; the gather structs + * and plumbing macros they use live in stats_internal.h. + */ + +/* mib must be prepared through "stats.arenas..extents" (depth 4). */ +static void +stats_gather_arena_extent(size_t *mib, unsigned j, stats_arena_extent_t *e) { + mib[4] = j; + CTL_LEAF(mib, 5, "ndirty", &e->ndirty, size_t); + CTL_LEAF(mib, 5, "nmuzzy", &e->nmuzzy, size_t); + CTL_LEAF(mib, 5, "nretained", &e->nretained, size_t); + CTL_LEAF(mib, 5, "npinned", &e->npinned, size_t); + CTL_LEAF(mib, 5, "dirty_bytes", &e->dirty_bytes, size_t); + CTL_LEAF(mib, 5, "muzzy_bytes", &e->muzzy_bytes, size_t); + CTL_LEAF(mib, 5, "retained_bytes", &e->retained_bytes, size_t); + CTL_LEAF(mib, 5, "pinned_bytes", &e->pinned_bytes, size_t); + e->ntotal = e->ndirty + e->nmuzzy + e->nretained + e->npinned; + e->total_bytes = e->dirty_bytes + e->muzzy_bytes + e->retained_bytes + + e->pinned_bytes; +} + +static void +stats_gather_arena_mem(unsigned i, stats_arena_mem_t *mem) { + CTL_M2_GET("stats.arenas.0.mapped", i, &mem->mapped, size_t); + CTL_M2_GET("stats.arenas.0.retained", i, &mem->retained, size_t); + CTL_M2_GET("stats.arenas.0.pinned", i, &mem->pinned, size_t); + CTL_M2_GET("stats.arenas.0.base", i, &mem->base, size_t); + CTL_M2_GET("stats.arenas.0.internal", i, &mem->internal, size_t); + CTL_M2_GET("stats.arenas.0.metadata_edata", i, &mem->metadata_edata, + size_t); + CTL_M2_GET("stats.arenas.0.metadata_rtree", i, &mem->metadata_rtree, + size_t); + CTL_M2_GET("stats.arenas.0.metadata_thp", i, &mem->metadata_thp, + size_t); + CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &mem->tcache_bytes, + size_t); + CTL_M2_GET("stats.arenas.0.tcache_stashed_bytes", i, + &mem->tcache_stashed_bytes, size_t); + CTL_M2_GET("stats.arenas.0.resident", i, &mem->resident, size_t); + CTL_M2_GET("stats.arenas.0.abandoned_vm", i, &mem->abandoned_vm, + size_t); + CTL_M2_GET("stats.arenas.0.extent_avail", i, &mem->extent_avail, + size_t); + CTL_GET("arenas.page", &mem->page, size_t); +} + +static void +stats_gather_arena_basics(unsigned i, stats_arena_basics_t *b) { + CTL_M2_GET("stats.arenas.0.nthreads", i, &b->nthreads, unsigned); + CTL_M2_GET("stats.arenas.0.uptime", i, &b->uptime, uint64_t); + CTL_M2_GET("stats.arenas.0.dss", i, &b->dss, const char *); + /* + * The arena..name ctl copies the name into the caller-provided + * buffer (b->name), so we hand it a real buffer rather than reading + * into a borrowed pointer. The merged/destroyed pseudo-arenas have no + * name, and querying them would fail, so guard the read. + */ + b->has_name = (i != MALLCTL_ARENAS_ALL && i != MALLCTL_ARENAS_DESTROYED); + if (b->has_name) { + char *namep = b->name; + CTL_M1_GET("arena.0.name", i, (void *)&namep, const char *); + } +} + +static void +stats_gather_arena_decay(unsigned i, stats_arena_decay_t *d) { + CTL_M2_GET( + "stats.arenas.0.dirty_decay_ms", i, &d->dirty_decay_ms, ssize_t); + CTL_M2_GET( + "stats.arenas.0.muzzy_decay_ms", i, &d->muzzy_decay_ms, ssize_t); + CTL_M2_GET("stats.arenas.0.pactive", i, &d->pactive, size_t); + CTL_M2_GET("stats.arenas.0.pdirty", i, &d->pdirty, size_t); + CTL_M2_GET("stats.arenas.0.pmuzzy", i, &d->pmuzzy, size_t); + CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &d->dirty_npurge, uint64_t); + CTL_M2_GET( + "stats.arenas.0.dirty_nmadvise", i, &d->dirty_nmadvise, uint64_t); + CTL_M2_GET("stats.arenas.0.dirty_purged", i, &d->dirty_purged, uint64_t); + CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &d->muzzy_npurge, uint64_t); + CTL_M2_GET( + "stats.arenas.0.muzzy_nmadvise", i, &d->muzzy_nmadvise, uint64_t); + CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &d->muzzy_purged, uint64_t); +} + +static void +stats_gather_arena_alloc(unsigned i, stats_arena_alloc_t *small, + stats_arena_alloc_t *large) { + CTL_M2_GET( + "stats.arenas.0.small.allocated", i, &small->allocated, size_t); + CTL_M2_GET("stats.arenas.0.small.nmalloc", i, &small->nmalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.small.ndalloc", i, &small->ndalloc, uint64_t); + CTL_M2_GET( + "stats.arenas.0.small.nrequests", i, &small->nrequests, uint64_t); + CTL_M2_GET("stats.arenas.0.small.nfills", i, &small->nfills, uint64_t); + CTL_M2_GET("stats.arenas.0.small.nflushes", i, &small->nflushes, + uint64_t); + CTL_M2_GET( + "stats.arenas.0.large.allocated", i, &large->allocated, size_t); + CTL_M2_GET("stats.arenas.0.large.nmalloc", i, &large->nmalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.large.ndalloc", i, &large->ndalloc, uint64_t); + CTL_M2_GET( + "stats.arenas.0.large.nrequests", i, &large->nrequests, uint64_t); + CTL_M2_GET("stats.arenas.0.large.nfills", i, &large->nfills, uint64_t); + CTL_M2_GET("stats.arenas.0.large.nflushes", i, &large->nflushes, + uint64_t); +} + +/* + * nslabs + prof (which gate the table gap/skip) are read by the caller; this + * reads the remaining detail fields. mibs prepared + indexed by the caller. + */ +static void +stats_gather_arena_bin(size_t *stats_mib, size_t *arenas_bin_mib, + stats_arena_bin_t *bin) { + CTL_LEAF(arenas_bin_mib, 3, "size", &bin->reg_size, size_t); + CTL_LEAF(arenas_bin_mib, 3, "nregs", &bin->nregs, uint32_t); + CTL_LEAF(arenas_bin_mib, 3, "slab_size", &bin->slab_size, size_t); + CTL_LEAF(arenas_bin_mib, 3, "nshards", &bin->nshards, uint32_t); + CTL_LEAF(stats_mib, 5, "nmalloc", &bin->nmalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "ndalloc", &bin->ndalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "curregs", &bin->curregs, size_t); + CTL_LEAF(stats_mib, 5, "nrequests", &bin->nrequests, uint64_t); + CTL_LEAF(stats_mib, 5, "nfills", &bin->nfills, uint64_t); + CTL_LEAF(stats_mib, 5, "nflushes", &bin->nflushes, uint64_t); + CTL_LEAF(stats_mib, 5, "nreslabs", &bin->nreslabs, uint64_t); + CTL_LEAF(stats_mib, 5, "curslabs", &bin->curslabs, size_t); + CTL_LEAF(stats_mib, 5, "nonfull_slabs", &bin->nonfull_slabs, size_t); +} + +static void +stats_gather_arena_lextent(size_t *stats_mib, size_t *arenas_lextent_mib, + size_t *prof_stats_mib, unsigned j, bool prof_stats_on, + stats_arena_lextent_t *lext) { + stats_mib[4] = j; + arenas_lextent_mib[2] = j; + CTL_LEAF(stats_mib, 5, "nmalloc", &lext->nmalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "ndalloc", &lext->ndalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "nrequests", &lext->nrequests, uint64_t); + CTL_LEAF(arenas_lextent_mib, 3, "size", &lext->lextent_size, size_t); + CTL_LEAF(stats_mib, 5, "curlextents", &lext->curlextents, size_t); + if (prof_stats_on) { + prof_stats_mib[3] = j; + CTL_LEAF(prof_stats_mib, 4, "live", &lext->prof_live, + prof_stats_t); + CTL_LEAF(prof_stats_mib, 4, "accum", &lext->prof_accum, + prof_stats_t); + } +} + +static void +stats_gather_arena_hpa_sec(unsigned i, stats_arena_hpa_sec_t *sec) { + CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec->sec_bytes, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec->sec_hits, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec->sec_misses, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i, + &sec->sec_dalloc_noflush, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, + &sec->sec_dalloc_flush, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_overfills", i, &sec->sec_overfills, + size_t); +} + +static void +stats_gather_arena_pac_sec(unsigned i, stats_arena_pac_sec_t *sec) { + CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec->sec_bytes, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec->sec_hits, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec->sec_misses, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i, + &sec->sec_dalloc_noflush, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i, + &sec->sec_dalloc_flush, size_t); +} + +static void +stats_gather_arena_hpa_counters(unsigned i, stats_arena_hpa_counters_t *c) { + CTL_M2_GET( + "stats.arenas.0.hpa_shard.npageslabs", i, &c->npageslabs, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.nactive", i, &c->nactive, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.ndirty", i, &c->ndirty, size_t); + + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_nonhuge", i, + &c->npageslabs_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_nonhuge", i, + &c->nactive_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_nonhuge", i, + &c->ndirty_nonhuge, size_t); + c->nretained_nonhuge = c->npageslabs_nonhuge * HUGEPAGE_PAGES + - c->nactive_nonhuge - c->ndirty_nonhuge; + + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_huge", i, + &c->npageslabs_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_huge", i, + &c->nactive_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_huge", i, + &c->ndirty_huge, size_t); + + CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes", i, + &c->npurge_passes, uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.npurges", i, &c->npurges, uint64_t); + CTL_M2_GET( + "stats.arenas.0.hpa_shard.nhugifies", i, &c->nhugifies, uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.nhugify_failures", i, + &c->nhugify_failures, uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.ndehugifies", i, &c->ndehugifies, + uint64_t); +} + +/* kind is "full_slabs" or "empty_slabs". */ +static void +stats_gather_arena_hpa_slab(unsigned i, const char *kind, + stats_arena_hpa_slab_t *s) { + size_t mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(mib, 0, "stats.arenas"); + mib[2] = i; + CTL_LEAF_PREPARE(mib, 3, "hpa_shard"); + CTL_LEAF_PREPARE(mib, 4, kind); + CTL_LEAF(mib, 5, "npageslabs_huge", &s->npageslabs_huge, size_t); + CTL_LEAF(mib, 5, "nactive_huge", &s->nactive_huge, size_t); + CTL_LEAF(mib, 5, "ndirty_huge", &s->ndirty_huge, size_t); + CTL_LEAF(mib, 5, "npageslabs_nonhuge", &s->npageslabs_nonhuge, size_t); + CTL_LEAF(mib, 5, "nactive_nonhuge", &s->nactive_nonhuge, size_t); + CTL_LEAF(mib, 5, "ndirty_nonhuge", &s->ndirty_nonhuge, size_t); + s->nretained_nonhuge = s->npageslabs_nonhuge * HUGEPAGE_PAGES + - s->nactive_nonhuge - s->ndirty_nonhuge; +} + +/* mib prepared through "stats.arenas..hpa_shard.nonfull_slabs" (depth 5). */ +static void +stats_gather_arena_hpa_nonfull(size_t *mib, unsigned j, + stats_arena_hpa_slab_t *s) { + mib[5] = j; + CTL_LEAF(mib, 6, "npageslabs_huge", &s->npageslabs_huge, size_t); + CTL_LEAF(mib, 6, "nactive_huge", &s->nactive_huge, size_t); + CTL_LEAF(mib, 6, "ndirty_huge", &s->ndirty_huge, size_t); + CTL_LEAF(mib, 6, "npageslabs_nonhuge", &s->npageslabs_nonhuge, size_t); + CTL_LEAF(mib, 6, "nactive_nonhuge", &s->nactive_nonhuge, size_t); + CTL_LEAF(mib, 6, "ndirty_nonhuge", &s->ndirty_nonhuge, size_t); + s->nretained_nonhuge = s->npageslabs_nonhuge * HUGEPAGE_PAGES + - s->nactive_nonhuge - s->ndirty_nonhuge; +} + +static void +stats_gather_arena_config(stats_arena_config_t *cfg) { + CTL_GET("arenas.narenas", &cfg->narenas, unsigned); + CTL_GET("arenas.dirty_decay_ms", &cfg->dirty_decay_ms, ssize_t); + CTL_GET("arenas.muzzy_decay_ms", &cfg->muzzy_decay_ms, ssize_t); + CTL_GET("arenas.quantum", &cfg->quantum, size_t); + CTL_GET("arenas.page", &cfg->page, size_t); + CTL_GET("arenas.hugepage", &cfg->hugepage, size_t); + size_t sz = sizeof(size_t); + cfg->have_tcache_max = (je_mallctl("arenas.tcache_max", + (void *)&cfg->tcache_max, &sz, NULL, 0) == 0); + CTL_GET("arenas.nbins", &cfg->nbins, unsigned); + CTL_GET("arenas.nhbins", &cfg->nhbins, unsigned); + CTL_GET("arenas.nlextents", &cfg->nlextents, unsigned); +} + +/* mib must be prepared through "arenas.bin" (depth 2). */ +static void +stats_gather_arena_bin_meta(size_t *mib, unsigned j, + stats_arena_bin_meta_t *bm) { + mib[2] = j; + CTL_LEAF(mib, 3, "size", &bm->size, size_t); + CTL_LEAF(mib, 3, "nregs", &bm->nregs, uint32_t); + CTL_LEAF(mib, 3, "slab_size", &bm->slab_size, size_t); + CTL_LEAF(mib, 3, "nshards", &bm->nshards, uint32_t); +} + +/* mib must be prepared through "arenas.lextent" (depth 2). */ +static void +stats_gather_arena_lextent_meta(size_t *mib, unsigned j, + stats_arena_lextent_meta_t *lm) { + mib[2] = j; + CTL_LEAF(mib, 3, "size", &lm->size, size_t); +} + +static void +stats_gather_global(stats_global_t *g) { + CTL_GET("stats.allocated", &g->allocated, size_t); + CTL_GET("stats.active", &g->active, size_t); + CTL_GET("stats.metadata", &g->metadata, size_t); + CTL_GET("stats.metadata_edata", &g->metadata_edata, size_t); + CTL_GET("stats.metadata_rtree", &g->metadata_rtree, size_t); + CTL_GET("stats.metadata_thp", &g->metadata_thp, size_t); + CTL_GET("stats.resident", &g->resident, size_t); + CTL_GET("stats.mapped", &g->mapped, size_t); + CTL_GET("stats.retained", &g->retained, size_t); + CTL_GET("stats.pinned", &g->pinned, size_t); + CTL_GET("stats.zero_reallocs", &g->zero_reallocs, size_t); + + if (have_background_thread) { + CTL_GET("stats.background_thread.num_threads", + &g->num_background_threads, size_t); + CTL_GET("stats.background_thread.num_runs", + &g->background_thread_num_runs, uint64_t); + CTL_GET("stats.background_thread.run_interval", + &g->background_thread_run_interval, uint64_t); + } else { + g->num_background_threads = 0; + g->background_thread_num_runs = 0; + g->background_thread_run_interval = 0; + } +} + +/* + * Fills initialized[narenas] and *destroyed_initialized; returns the number of + * initialized arenas. + */ +static unsigned +stats_gather_arenas_initialized(unsigned narenas, bool *initialized, + bool *destroyed_initialized) { + size_t mib[3]; + size_t miblen = sizeof(mib) / sizeof(size_t); + size_t sz; + unsigned ninitialized = 0; + + xmallctlnametomib("arena.0.initialized", mib, &miblen); + for (unsigned i = 0; i < narenas; i++) { + mib[1] = i; + sz = sizeof(bool); + xmallctlbymib(mib, miblen, &initialized[i], &sz, NULL, 0); + if (initialized[i]) { + ninitialized++; + } + } + mib[1] = MALLCTL_ARENAS_DESTROYED; + sz = sizeof(bool); + xmallctlbymib(mib, miblen, destroyed_initialized, &sz, NULL, 0); + return ninitialized; +} + /******************************************************************************/ static uint64_t @@ -273,42 +702,20 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, #undef EMITTER_TYPE_uint64_t } -#define COL_DECLARE(column_name) emitter_col_t col_##column_name; - -#define COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ - emitter_col_init(&col_##column_name, &row_name); \ - col_##column_name.justify = emitter_justify_##left_or_right; \ - col_##column_name.width = col_width; \ - col_##column_name.type = emitter_type_##etype; - -#define COL(row_name, column_name, left_or_right, col_width, etype) \ - COL_DECLARE(column_name); \ - COL_INIT(row_name, column_name, left_or_right, col_width, etype) - -#define COL_HDR_DECLARE(column_name) \ - COL_DECLARE(column_name); \ - emitter_col_t header_##column_name; - -#define COL_HDR_INIT( \ - row_name, column_name, human, left_or_right, col_width, etype) \ - COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ - emitter_col_init(&header_##column_name, &header_##row_name); \ - header_##column_name.justify = emitter_justify_##left_or_right; \ - header_##column_name.width = col_width; \ - header_##column_name.type = emitter_type_title; \ - header_##column_name.str_val = human ? human : #column_name; - -#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ - COL_HDR_DECLARE(column_name) \ - COL_HDR_INIT( \ - row_name, column_name, human, left_or_right, col_width, etype) - JEMALLOC_COLD static void stats_arena_bins_print( emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { + /* + * Streaming table (deviates from the gather/emit split): the SC_NBINS + * rows are gathered and emitted one at a time (stats_gather_arena_bin + * per row + inline emit, with all-zero runs collapsed to "---"). This + * preserves O(1) memory. Full gather-all-rows-before-emit separation + * would require buffering the table; a separate per-row emit helper + * would not. The lextents / extents / hpa-nonfull tables follow the + * same pattern. + */ size_t page; - bool in_gap, in_gap_prev; unsigned nbins, j; CTL_GET("arenas.page", &page, size_t); @@ -399,61 +806,44 @@ stats_arena_bins_print( CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.bins"); } - for (j = 0, in_gap = false; j < nbins; j++) { - uint64_t nslabs; - size_t reg_size, slab_size, curregs; - size_t curslabs; - size_t nonfull_slabs; - uint32_t nregs, nshards; - uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; - uint64_t nreslabs; - prof_stats_t prof_live; - prof_stats_t prof_accum; - + emitter_table_sparse_begin(emitter); + for (j = 0; j < nbins; j++) { + stats_arena_bin_t bin; stats_arenas_mib[4] = j; arenas_bin_mib[2] = j; - CTL_LEAF(stats_arenas_mib, 5, "nslabs", &nslabs, uint64_t); + CTL_LEAF(stats_arenas_mib, 5, "nslabs", &bin.nslabs, uint64_t); if (prof_stats_on) { prof_stats_mib[3] = j; - CTL_LEAF(prof_stats_mib, 4, "live", &prof_live, + CTL_LEAF(prof_stats_mib, 4, "live", &bin.prof_live, prof_stats_t); - CTL_LEAF(prof_stats_mib, 4, "accum", &prof_accum, + CTL_LEAF(prof_stats_mib, 4, "accum", &bin.prof_accum, prof_stats_t); } - in_gap_prev = in_gap; + bool is_gap; if (prof_stats_on) { - in_gap = (nslabs == 0 && prof_accum.count == 0); + is_gap = (bin.nslabs == 0 && bin.prof_accum.count == 0); } else { - in_gap = (nslabs == 0); + is_gap = (bin.nslabs == 0); } - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } - - if (in_gap && !emitter_outputs_json(emitter)) { + if (is_gap && !emitter_outputs_json(emitter)) { + emitter_table_sparse_row(emitter, &row, true); continue; } - CTL_LEAF(arenas_bin_mib, 3, "size", ®_size, size_t); - CTL_LEAF(arenas_bin_mib, 3, "nregs", &nregs, uint32_t); - CTL_LEAF(arenas_bin_mib, 3, "slab_size", &slab_size, size_t); - CTL_LEAF(arenas_bin_mib, 3, "nshards", &nshards, uint32_t); - CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "curregs", &curregs, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "nrequests", &nrequests, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "nfills", &nfills, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "nflushes", &nflushes, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "nreslabs", &nreslabs, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "curslabs", &curslabs, size_t); - CTL_LEAF(stats_arenas_mib, 5, "nonfull_slabs", &nonfull_slabs, - size_t); + /* + * Per-size-class tables stream: gather one row, emit it, reuse + * buffers. Full gather-all-rows-before-emit separation would + * require table-sized storage, so gathering and emission remain + * interleaved across rows. In particular the per-bin mutex + * counters are read here (the mutex subsystem reads straight into + * emitter columns) rather than up front. A separate per-row emit + * helper could further decouple the code without buffering. + */ + stats_gather_arena_bin(stats_arenas_mib, arenas_bin_mib, &bin); if (mutex) { mutex_stats_read_arena_bin(stats_arenas_mib, 5, @@ -462,33 +852,33 @@ stats_arena_bins_print( emitter_json_object_begin(emitter); emitter_json_kv( - emitter, "nmalloc", emitter_type_uint64, &nmalloc); + emitter, "nmalloc", emitter_type_uint64, &bin.nmalloc); emitter_json_kv( - emitter, "ndalloc", emitter_type_uint64, &ndalloc); + emitter, "ndalloc", emitter_type_uint64, &bin.ndalloc); emitter_json_kv( - emitter, "curregs", emitter_type_size, &curregs); + emitter, "curregs", emitter_type_size, &bin.curregs); emitter_json_kv( - emitter, "nrequests", emitter_type_uint64, &nrequests); + emitter, "nrequests", emitter_type_uint64, &bin.nrequests); if (prof_stats_on) { emitter_json_kv(emitter, "prof_live_requested", - emitter_type_uint64, &prof_live.req_sum); + emitter_type_uint64, &bin.prof_live.req_sum); emitter_json_kv(emitter, "prof_live_count", - emitter_type_uint64, &prof_live.count); + emitter_type_uint64, &bin.prof_live.count); emitter_json_kv(emitter, "prof_accum_requested", - emitter_type_uint64, &prof_accum.req_sum); + emitter_type_uint64, &bin.prof_accum.req_sum); emitter_json_kv(emitter, "prof_accum_count", - emitter_type_uint64, &prof_accum.count); + emitter_type_uint64, &bin.prof_accum.count); } emitter_json_kv( - emitter, "nfills", emitter_type_uint64, &nfills); + emitter, "nfills", emitter_type_uint64, &bin.nfills); emitter_json_kv( - emitter, "nflushes", emitter_type_uint64, &nflushes); + emitter, "nflushes", emitter_type_uint64, &bin.nflushes); emitter_json_kv( - emitter, "nreslabs", emitter_type_uint64, &nreslabs); + emitter, "nreslabs", emitter_type_uint64, &bin.nreslabs); emitter_json_kv( - emitter, "curslabs", emitter_type_size, &curslabs); + emitter, "curslabs", emitter_type_size, &bin.curslabs); emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size, - &nonfull_slabs); + &bin.nonfull_slabs); if (mutex) { emitter_json_object_kv_begin(emitter, "mutex"); mutex_stats_emit( @@ -497,13 +887,13 @@ stats_arena_bins_print( } emitter_json_object_end(emitter); - size_t availregs = nregs * curslabs; + size_t availregs = bin.nregs * bin.curslabs; char util[6]; if (get_rate_str( - (uint64_t)curregs, (uint64_t)availregs, util)) { + (uint64_t)bin.curregs, (uint64_t)availregs, util)) { if (availregs == 0) { malloc_snprintf(util, sizeof(util), "1"); - } else if (curregs > availregs) { + } else if (bin.curregs > availregs) { /* * Race detected: the counters were read in * separate mallctl calls and concurrent @@ -516,57 +906,55 @@ stats_arena_bins_print( } } - col_size.size_val = reg_size; + col_size.size_val = bin.reg_size; col_ind.unsigned_val = j; - col_allocated.size_val = curregs * reg_size; - col_nmalloc.uint64_val = nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); - col_ndalloc.uint64_val = ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); - col_nrequests.uint64_val = nrequests; + col_allocated.size_val = bin.curregs * bin.reg_size; + col_nmalloc.uint64_val = bin.nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(bin.nmalloc, uptime); + col_ndalloc.uint64_val = bin.ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(bin.ndalloc, uptime); + col_nrequests.uint64_val = bin.nrequests; col_nrequests_ps.uint64_val = rate_per_second( - nrequests, uptime); + bin.nrequests, uptime); if (prof_stats_on) { - col_prof_live_requested.uint64_val = prof_live.req_sum; - col_prof_live_count.uint64_val = prof_live.count; + col_prof_live_requested.uint64_val = bin.prof_live.req_sum; + col_prof_live_count.uint64_val = bin.prof_live.count; col_prof_accum_requested.uint64_val = - prof_accum.req_sum; - col_prof_accum_count.uint64_val = prof_accum.count; + bin.prof_accum.req_sum; + col_prof_accum_count.uint64_val = bin.prof_accum.count; } - col_nshards.unsigned_val = nshards; - col_curregs.size_val = curregs; - col_curslabs.size_val = curslabs; - col_nonfull_slabs.size_val = nonfull_slabs; - col_regs.unsigned_val = nregs; - col_pgs.size_val = slab_size / page; + col_nshards.unsigned_val = bin.nshards; + col_curregs.size_val = bin.curregs; + col_curslabs.size_val = bin.curslabs; + col_nonfull_slabs.size_val = bin.nonfull_slabs; + col_regs.unsigned_val = bin.nregs; + col_pgs.size_val = bin.slab_size / page; col_util.str_val = util; - col_nfills.uint64_val = nfills; - col_nfills_ps.uint64_val = rate_per_second(nfills, uptime); - col_nflushes.uint64_val = nflushes; - col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime); - col_nslabs.uint64_val = nslabs; - col_nreslabs.uint64_val = nreslabs; - col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime); + col_nfills.uint64_val = bin.nfills; + col_nfills_ps.uint64_val = rate_per_second(bin.nfills, uptime); + col_nflushes.uint64_val = bin.nflushes; + col_nflushes_ps.uint64_val = rate_per_second(bin.nflushes, uptime); + col_nslabs.uint64_val = bin.nslabs; + col_nreslabs.uint64_val = bin.nreslabs; + col_nreslabs_ps.uint64_val = rate_per_second(bin.nreslabs, uptime); /* * Note that mutex columns were initialized above, if mutex == * true. */ - emitter_table_row(emitter, &row); + emitter_table_sparse_row(emitter, &row, is_gap); } emitter_json_array_end(emitter); /* Close "bins". */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); } JEMALLOC_COLD static void stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { + /* Streaming table; see stats_arena_bins_print for the rationale. */ unsigned nbins, nlextents, j; - bool in_gap, in_gap_prev; CTL_GET("arenas.nbins", &nbins, unsigned); CTL_GET("arenas.nlextents", &nlextents, unsigned); @@ -597,8 +985,13 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { COL_HDR_INIT(row, prof_live_count, NULL, right, 17, uint64) COL_HDR_INIT(row, prof_accum_requested, NULL, right, 21, uint64) COL_HDR_INIT(row, prof_accum_count, NULL, right, 17, uint64) + col_prof_live_requested.json_key = "prof_live_requested"; + col_prof_live_count.json_key = "prof_live_count"; + col_prof_accum_requested.json_key = "prof_accum_requested"; + col_prof_accum_count.json_key = "prof_accum_count"; } COL_HDR(row, curlextents, NULL, right, 13, size) + col_curlextents.json_key = "curlextents"; /* As with bins, we label the large extents table. */ header_size.width -= 6; @@ -619,89 +1012,46 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.lextents"); } - for (j = 0, in_gap = false; j < nlextents; j++) { - uint64_t nmalloc, ndalloc, nrequests; - size_t lextent_size, curlextents; - prof_stats_t prof_live; - prof_stats_t prof_accum; + emitter_table_sparse_begin(emitter); + for (j = 0; j < nlextents; j++) { + stats_arena_lextent_t lext; + stats_gather_arena_lextent(stats_arenas_mib, arenas_lextent_mib, + prof_stats_mib, j, prof_stats_on, &lext); - stats_arenas_mib[4] = j; - arenas_lextent_mib[2] = j; - - CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t); - CTL_LEAF( - stats_arenas_mib, 5, "nrequests", &nrequests, uint64_t); - - in_gap_prev = in_gap; - in_gap = (nrequests == 0); - - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } - - CTL_LEAF(arenas_lextent_mib, 3, "size", &lextent_size, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "curlextents", &curlextents, size_t); + bool is_gap = (lext.nrequests == 0); + col_size.size_val = lext.lextent_size; + col_ind.unsigned_val = nbins + j; + col_allocated.size_val = lext.curlextents * lext.lextent_size; + col_nmalloc.uint64_val = lext.nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(lext.nmalloc, uptime); + col_ndalloc.uint64_val = lext.ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(lext.ndalloc, uptime); + col_nrequests.uint64_val = lext.nrequests; + col_nrequests_ps.uint64_val = rate_per_second( + lext.nrequests, uptime); if (prof_stats_on) { - prof_stats_mib[3] = j; - CTL_LEAF(prof_stats_mib, 4, "live", &prof_live, - prof_stats_t); - CTL_LEAF(prof_stats_mib, 4, "accum", &prof_accum, - prof_stats_t); + col_prof_live_requested.uint64_val = lext.prof_live.req_sum; + col_prof_live_count.uint64_val = lext.prof_live.count; + col_prof_accum_requested.uint64_val = + lext.prof_accum.req_sum; + col_prof_accum_count.uint64_val = lext.prof_accum.count; } + col_curlextents.size_val = lext.curlextents; emitter_json_object_begin(emitter); - if (prof_stats_on) { - emitter_json_kv(emitter, "prof_live_requested", - emitter_type_uint64, &prof_live.req_sum); - emitter_json_kv(emitter, "prof_live_count", - emitter_type_uint64, &prof_live.count); - emitter_json_kv(emitter, "prof_accum_requested", - emitter_type_uint64, &prof_accum.req_sum); - emitter_json_kv(emitter, "prof_accum_count", - emitter_type_uint64, &prof_accum.count); - } - emitter_json_kv( - emitter, "curlextents", emitter_type_size, &curlextents); + emitter_sparse_row(emitter, &row, is_gap); emitter_json_object_end(emitter); - - col_size.size_val = lextent_size; - col_ind.unsigned_val = nbins + j; - col_allocated.size_val = curlextents * lextent_size; - col_nmalloc.uint64_val = nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); - col_ndalloc.uint64_val = ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); - col_nrequests.uint64_val = nrequests; - col_nrequests_ps.uint64_val = rate_per_second( - nrequests, uptime); - if (prof_stats_on) { - col_prof_live_requested.uint64_val = prof_live.req_sum; - col_prof_live_count.uint64_val = prof_live.count; - col_prof_accum_requested.uint64_val = - prof_accum.req_sum; - col_prof_accum_count.uint64_val = prof_accum.count; - } - col_curlextents.size_val = curlextents; - - if (!in_gap) { - emitter_table_row(emitter, &row); - } } emitter_json_array_end(emitter); /* Close "lextents". */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); } JEMALLOC_COLD static void stats_arena_extents_print(emitter_t *emitter, unsigned i) { + /* Streaming table; see stats_arena_bins_print for the rationale. */ unsigned j; - bool in_gap, in_gap_prev; emitter_row_t header_row; emitter_row_init(&header_row); emitter_row_t row; @@ -731,197 +1081,103 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { stats_arenas_mib[2] = i; CTL_LEAF_PREPARE(stats_arenas_mib, 3, "extents"); - in_gap = false; + emitter_table_sparse_begin(emitter); for (j = 0; j < SC_NPSIZES; j++) { - size_t ndirty, nmuzzy, nretained, npinned, total, - dirty_bytes, muzzy_bytes, retained_bytes, pinned_bytes, - total_bytes; - stats_arenas_mib[4] = j; + stats_arena_extent_t e; + stats_gather_arena_extent(stats_arenas_mib, j, &e); - CTL_LEAF(stats_arenas_mib, 5, "ndirty", &ndirty, size_t); - CTL_LEAF(stats_arenas_mib, 5, "nmuzzy", &nmuzzy, size_t); - CTL_LEAF(stats_arenas_mib, 5, "nretained", &nretained, size_t); - CTL_LEAF(stats_arenas_mib, 5, "npinned", &npinned, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "dirty_bytes", &dirty_bytes, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "muzzy_bytes", &muzzy_bytes, size_t); - CTL_LEAF(stats_arenas_mib, 5, "retained_bytes", &retained_bytes, - size_t); - CTL_LEAF(stats_arenas_mib, 5, "pinned_bytes", &pinned_bytes, - size_t); - - total = ndirty + nmuzzy + nretained + npinned; - total_bytes = dirty_bytes + muzzy_bytes + retained_bytes - + pinned_bytes; - - in_gap_prev = in_gap; - in_gap = (total == 0); - - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } + bool is_gap = (e.ntotal == 0); emitter_json_object_begin(emitter); - emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); - emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy); + emitter_json_kv(emitter, "ndirty", emitter_type_size, &e.ndirty); + emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &e.nmuzzy); emitter_json_kv( - emitter, "nretained", emitter_type_size, &nretained); + emitter, "nretained", emitter_type_size, &e.nretained); emitter_json_kv( - emitter, "npinned", emitter_type_size, &npinned); + emitter, "npinned", emitter_type_size, &e.npinned); emitter_json_kv( - emitter, "dirty_bytes", emitter_type_size, &dirty_bytes); + emitter, "dirty_bytes", emitter_type_size, &e.dirty_bytes); emitter_json_kv( - emitter, "muzzy_bytes", emitter_type_size, &muzzy_bytes); + emitter, "muzzy_bytes", emitter_type_size, &e.muzzy_bytes); emitter_json_kv(emitter, "retained_bytes", emitter_type_size, - &retained_bytes); + &e.retained_bytes); emitter_json_kv(emitter, "pinned_bytes", emitter_type_size, - &pinned_bytes); + &e.pinned_bytes); emitter_json_object_end(emitter); col_size.size_val = sz_pind2sz(j); col_ind.size_val = j; - col_ndirty.size_val = ndirty; - col_dirty.size_val = dirty_bytes; - col_nmuzzy.size_val = nmuzzy; - col_muzzy.size_val = muzzy_bytes; - col_nretained.size_val = nretained; - col_retained.size_val = retained_bytes; - col_npinned.size_val = npinned; - col_pinned.size_val = pinned_bytes; - col_ntotal.size_val = total; - col_total.size_val = total_bytes; + col_ndirty.size_val = e.ndirty; + col_dirty.size_val = e.dirty_bytes; + col_nmuzzy.size_val = e.nmuzzy; + col_muzzy.size_val = e.muzzy_bytes; + col_nretained.size_val = e.nretained; + col_retained.size_val = e.retained_bytes; + col_npinned.size_val = e.npinned; + col_pinned.size_val = e.pinned_bytes; + col_ntotal.size_val = e.ntotal; + col_total.size_val = e.total_bytes; - if (!in_gap) { - emitter_table_row(emitter, &row); - } + emitter_table_sparse_row(emitter, &row, is_gap); } emitter_json_array_end(emitter); /* Close "extents". */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); +} + +static void +stats_emit_arena_hpa_sec(emitter_t *emitter, const stats_arena_hpa_sec_t *sec) { + emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache", + emitter_type_size, &sec->sec_bytes); + emitter_kv(emitter, "sec_hits", "Total hits in small extent cache", + emitter_type_size, &sec->sec_hits); + emitter_kv(emitter, "sec_misses", "Total misses in small extent cache", + emitter_type_size, &sec->sec_misses); + emitter_kv(emitter, "sec_dalloc_noflush", + "Dalloc calls without flush in small extent cache", + emitter_type_size, &sec->sec_dalloc_noflush); + emitter_kv(emitter, "sec_dalloc_flush", + "Dalloc calls with flush in small extent cache", emitter_type_size, + &sec->sec_dalloc_flush); + emitter_kv(emitter, "sec_overfills", + "sec_fill calls that went over max_bytes", emitter_type_size, + &sec->sec_overfills); } static void stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) { - size_t sec_bytes; - size_t sec_hits; - size_t sec_misses; - size_t sec_dalloc_flush; - size_t sec_dalloc_noflush; - size_t sec_overfills; - CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t); - emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache", - emitter_type_size, &sec_bytes); - CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec_hits, size_t); - emitter_kv(emitter, "sec_hits", "Total hits in small extent cache", - emitter_type_size, &sec_hits); - CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec_misses, size_t); - emitter_kv(emitter, "sec_misses", "Total misses in small extent cache", - emitter_type_size, &sec_misses); - CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i, - &sec_dalloc_noflush, size_t); - emitter_kv(emitter, "sec_dalloc_noflush", - "Dalloc calls without flush in small extent cache", - emitter_type_size, &sec_dalloc_noflush); - CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, &sec_dalloc_flush, - size_t); - emitter_kv(emitter, "sec_dalloc_flush", - "Dalloc calls with flush in small extent cache", emitter_type_size, - &sec_dalloc_flush); - CTL_M2_GET( - "stats.arenas.0.hpa_sec_overfills", i, &sec_overfills, size_t); - emitter_kv(emitter, "sec_overfills", - "sec_fill calls that went over max_bytes", emitter_type_size, - &sec_overfills); + stats_arena_hpa_sec_t sec; + stats_gather_arena_hpa_sec(i, &sec); + stats_emit_arena_hpa_sec(emitter, &sec); +} + +static void +stats_emit_arena_pac_sec(emitter_t *emitter, const stats_arena_pac_sec_t *sec) { + emitter_kv(emitter, "pac_sec_bytes", "Bytes in PAC small extent cache", + emitter_type_size, &sec->sec_bytes); + emitter_kv(emitter, "pac_sec_hits", "Total hits in PAC small extent cache", + emitter_type_size, &sec->sec_hits); + emitter_kv(emitter, "pac_sec_misses", + "Total misses in PAC small extent cache", emitter_type_size, + &sec->sec_misses); + emitter_kv(emitter, "pac_sec_dalloc_noflush", + "Dalloc calls without flush in PAC small extent cache", + emitter_type_size, &sec->sec_dalloc_noflush); + emitter_kv(emitter, "pac_sec_dalloc_flush", + "Dalloc calls with flush in PAC small extent cache", + emitter_type_size, &sec->sec_dalloc_flush); } static void stats_arena_pac_sec_print(emitter_t *emitter, unsigned i) { - size_t sec_bytes; - size_t sec_hits; - size_t sec_misses; - size_t sec_dalloc_flush; - size_t sec_dalloc_noflush; - CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec_bytes, size_t); - emitter_kv(emitter, "pac_sec_bytes", - "Bytes in PAC small extent cache", - emitter_type_size, &sec_bytes); - CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec_hits, size_t); - emitter_kv(emitter, "pac_sec_hits", - "Total hits in PAC small extent cache", - emitter_type_size, &sec_hits); - CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec_misses, size_t); - emitter_kv(emitter, "pac_sec_misses", - "Total misses in PAC small extent cache", - emitter_type_size, &sec_misses); - CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i, - &sec_dalloc_noflush, size_t); - emitter_kv(emitter, "pac_sec_dalloc_noflush", - "Dalloc calls without flush in PAC small extent cache", - emitter_type_size, &sec_dalloc_noflush); - CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i, &sec_dalloc_flush, - size_t); - emitter_kv(emitter, "pac_sec_dalloc_flush", - "Dalloc calls with flush in PAC small extent cache", - emitter_type_size, &sec_dalloc_flush); + stats_arena_pac_sec_t sec; + stats_gather_arena_pac_sec(i, &sec); + stats_emit_arena_pac_sec(emitter, &sec); } static void -stats_arena_hpa_shard_counters_print( - emitter_t *emitter, unsigned i, uint64_t uptime) { - size_t npageslabs; - size_t nactive; - size_t ndirty; - - size_t npageslabs_nonhuge; - size_t nactive_nonhuge; - size_t ndirty_nonhuge; - size_t nretained_nonhuge; - - size_t npageslabs_huge; - size_t nactive_huge; - size_t ndirty_huge; - - uint64_t npurge_passes; - uint64_t npurges; - uint64_t nhugifies; - uint64_t nhugify_failures; - uint64_t ndehugifies; - - CTL_M2_GET( - "stats.arenas.0.hpa_shard.npageslabs", i, &npageslabs, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.nactive", i, &nactive, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.ndirty", i, &ndirty, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_nonhuge", i, - &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_nonhuge", i, - &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_nonhuge", i, - &ndirty_nonhuge, size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_huge", i, - &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_huge", i, - &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_huge", i, - &ndirty_huge, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes", i, &npurge_passes, - uint64_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.npurges", i, &npurges, uint64_t); - CTL_M2_GET( - "stats.arenas.0.hpa_shard.nhugifies", i, &nhugifies, uint64_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.nhugify_failures", i, - &nhugify_failures, uint64_t); - CTL_M2_GET( - "stats.arenas.0.hpa_shard.ndehugifies", i, &ndehugifies, uint64_t); - +stats_emit_arena_hpa_counters(emitter_t *emitter, + const stats_arena_hpa_counters_t *c, uint64_t uptime) { emitter_table_printf(emitter, "HPA shard stats:\n" " Pageslabs: %zu (%zu huge, %zu nonhuge)\n" @@ -937,47 +1193,60 @@ stats_arena_hpa_shard_counters_print( " Hugify failures: %" FMTu64 " (%" FMTu64 " / sec)\n" " Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n", - npageslabs, npageslabs_huge, npageslabs_nonhuge, nactive, - nactive_huge, nactive_nonhuge, ndirty, ndirty_huge, ndirty_nonhuge, - nretained_nonhuge, npurge_passes, - rate_per_second(npurge_passes, uptime), npurges, - rate_per_second(npurges, uptime), nhugifies, - rate_per_second(nhugifies, uptime), nhugify_failures, - rate_per_second(nhugify_failures, uptime), ndehugifies, - rate_per_second(ndehugifies, uptime)); + c->npageslabs, c->npageslabs_huge, c->npageslabs_nonhuge, c->nactive, + c->nactive_huge, c->nactive_nonhuge, c->ndirty, c->ndirty_huge, + c->ndirty_nonhuge, c->nretained_nonhuge, c->npurge_passes, + rate_per_second(c->npurge_passes, uptime), c->npurges, + rate_per_second(c->npurges, uptime), c->nhugifies, + rate_per_second(c->nhugifies, uptime), c->nhugify_failures, + rate_per_second(c->nhugify_failures, uptime), c->ndehugifies, + rate_per_second(c->ndehugifies, uptime)); - emitter_json_kv(emitter, "npageslabs", emitter_type_size, &npageslabs); - emitter_json_kv(emitter, "nactive", emitter_type_size, &nactive); - emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); + emitter_json_kv(emitter, "npageslabs", emitter_type_size, &c->npageslabs); + emitter_json_kv(emitter, "nactive", emitter_type_size, &c->nactive); + emitter_json_kv(emitter, "ndirty", emitter_type_size, &c->ndirty); emitter_json_kv( - emitter, "npurge_passes", emitter_type_uint64, &npurge_passes); - emitter_json_kv(emitter, "npurges", emitter_type_uint64, &npurges); - emitter_json_kv(emitter, "nhugifies", emitter_type_uint64, &nhugifies); + emitter, "npurge_passes", emitter_type_uint64, &c->npurge_passes); + emitter_json_kv(emitter, "npurges", emitter_type_uint64, &c->npurges); + emitter_json_kv(emitter, "nhugifies", emitter_type_uint64, &c->nhugifies); emitter_json_kv(emitter, "nhugify_failures", emitter_type_uint64, - &nhugify_failures); + &c->nhugify_failures); emitter_json_kv( - emitter, "ndehugifies", emitter_type_uint64, &ndehugifies); + emitter, "ndehugifies", emitter_type_uint64, &c->ndehugifies); emitter_json_object_kv_begin(emitter, "slabs"); emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &npageslabs_nonhuge); + &c->npageslabs_nonhuge); emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); + emitter, "nactive_nonhuge", emitter_type_size, &c->nactive_nonhuge); emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); + emitter, "ndirty_nonhuge", emitter_type_size, &c->ndirty_nonhuge); emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); + &c->nretained_nonhuge); emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); + emitter, "npageslabs_huge", emitter_type_size, &c->npageslabs_huge); emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter, "nactive_huge", emitter_type_size, &c->nactive_huge); emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); + emitter, "ndirty_huge", emitter_type_size, &c->ndirty_huge); emitter_json_object_end(emitter); /* End "slabs" */ +} - /* alloc_batch stats */ +static void +stats_arena_hpa_shard_counters_print( + emitter_t *emitter, unsigned i, uint64_t uptime) { + stats_arena_hpa_counters_t c; + stats_gather_arena_hpa_counters(i, &c); + stats_emit_arena_hpa_counters(emitter, &c, uptime); + + /* + * The extent-allocation distribution below is a small fixed-size + * (SEC_MAX_NALLOCS + 1) table with its own leaf ctls, so it is + * gathered into stack arrays and emitted (table then JSON) inline + * rather than through a stats_gather/stats_emit pair. + */ uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1]; uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1]; uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1]; @@ -1054,6 +1323,37 @@ stats_arena_hpa_shard_counters_print( emitter_json_array_end(emitter); /* End "alloc_batch" */ } +static void +stats_emit_arena_hpa_slab(emitter_t *emitter, const stats_arena_hpa_slab_t *s, + const char *label, const char *json_key) { + emitter_table_printf(emitter, + " In %s slabs:\n" + " npageslabs: %zu huge, %zu nonhuge\n" + " nactive: %zu huge, %zu nonhuge \n" + " ndirty: %zu huge, %zu nonhuge \n" + " nretained: 0 huge, %zu nonhuge \n", + label, s->npageslabs_huge, s->npageslabs_nonhuge, s->nactive_huge, + s->nactive_nonhuge, s->ndirty_huge, s->ndirty_nonhuge, + s->nretained_nonhuge); + + emitter_json_object_kv_begin(emitter, json_key); + emitter_json_kv( + emitter, "npageslabs_huge", emitter_type_size, &s->npageslabs_huge); + emitter_json_kv( + emitter, "nactive_huge", emitter_type_size, &s->nactive_huge); + emitter_json_kv( + emitter, "ndirty_huge", emitter_type_size, &s->ndirty_huge); + emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, + &s->npageslabs_nonhuge); + emitter_json_kv( + emitter, "nactive_nonhuge", emitter_type_size, &s->nactive_nonhuge); + emitter_json_kv( + emitter, "ndirty_nonhuge", emitter_type_size, &s->ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &s->nretained_nonhuge); + emitter_json_object_end(emitter); /* End "full_slabs"/"empty_slabs" */ +} + static void stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_row_t header_row; @@ -1061,102 +1361,20 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_row_t row; emitter_row_init(&row); - size_t npageslabs_huge; - size_t nactive_huge; - size_t ndirty_huge; + stats_arena_hpa_slab_t sfull; + stats_gather_arena_hpa_slab(i, "full_slabs", &sfull); + stats_emit_arena_hpa_slab(emitter, &sfull, "full", "full_slabs"); - size_t npageslabs_nonhuge; - size_t nactive_nonhuge; - size_t ndirty_nonhuge; - size_t nretained_nonhuge; + stats_arena_hpa_slab_t sempty; + stats_gather_arena_hpa_slab(i, "empty_slabs", &sempty); + stats_emit_arena_hpa_slab(emitter, &sempty, "empty", "empty_slabs"); - /* Full slab stats. */ - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_huge", i, - &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_huge", i, - &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_huge", i, - &ndirty_huge, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_nonhuge", i, - &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_nonhuge", i, - &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_nonhuge", i, - &ndirty_nonhuge, size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - emitter_table_printf(emitter, - " In full slabs:\n" - " npageslabs: %zu huge, %zu nonhuge\n" - " nactive: %zu huge, %zu nonhuge \n" - " ndirty: %zu huge, %zu nonhuge \n" - " nretained: 0 huge, %zu nonhuge \n", - npageslabs_huge, npageslabs_nonhuge, nactive_huge, nactive_nonhuge, - ndirty_huge, ndirty_nonhuge, nretained_nonhuge); - - emitter_json_object_kv_begin(emitter, "full_slabs"); - emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &npageslabs_nonhuge); - emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); - emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); - emitter_json_object_end(emitter); /* End "full_slabs" */ - - /* Next, empty slab stats. */ - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_huge", i, - &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_huge", i, - &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_huge", i, - &ndirty_huge, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_nonhuge", i, - &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_nonhuge", i, - &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_nonhuge", i, - &ndirty_nonhuge, size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - emitter_table_printf(emitter, - " In empty slabs:\n" - " npageslabs: %zu huge, %zu nonhuge\n" - " nactive: %zu huge, %zu nonhuge \n" - " ndirty: %zu huge, %zu nonhuge \n" - " nretained: 0 huge, %zu nonhuge \n", - npageslabs_huge, npageslabs_nonhuge, nactive_huge, nactive_nonhuge, - ndirty_huge, ndirty_nonhuge, nretained_nonhuge); - - emitter_json_object_kv_begin(emitter, "empty_slabs"); - emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &npageslabs_nonhuge); - emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); - emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); - emitter_json_object_end(emitter); /* End "empty_slabs" */ - - /* Last, nonfull slab stats. */ + /* + * Nonfull slabs: a per-size-class streaming table (gather one row, + * emit one row, collapsing all-zero-row runs to "---"), so gather and + * emit are necessarily interleaved -- like the bins/lextents/extents + * tables. + */ COL_HDR(row, size, NULL, right, 20, size) COL_HDR(row, ind, NULL, right, 4, unsigned) COL_HDR(row, npageslabs_huge, NULL, right, 16, size) @@ -1175,67 +1393,43 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_table_printf(emitter, " In nonfull slabs:\n"); emitter_table_row(emitter, &header_row); emitter_json_array_kv_begin(emitter, "nonfull_slabs"); - bool in_gap = false; + emitter_table_sparse_begin(emitter); for (pszind_t j = 0; j < PSSET_NPSIZES && j < SC_NPSIZES; j++) { - stats_arenas_mib[5] = j; + stats_arena_hpa_slab_t s; + stats_gather_arena_hpa_nonfull(stats_arenas_mib, j, &s); - CTL_LEAF(stats_arenas_mib, 6, "npageslabs_huge", - &npageslabs_huge, size_t); - CTL_LEAF( - stats_arenas_mib, 6, "nactive_huge", &nactive_huge, size_t); - CTL_LEAF( - stats_arenas_mib, 6, "ndirty_huge", &ndirty_huge, size_t); - - CTL_LEAF(stats_arenas_mib, 6, "npageslabs_nonhuge", - &npageslabs_nonhuge, size_t); - CTL_LEAF(stats_arenas_mib, 6, "nactive_nonhuge", - &nactive_nonhuge, size_t); - CTL_LEAF(stats_arenas_mib, 6, "ndirty_nonhuge", &ndirty_nonhuge, - size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - bool in_gap_prev = in_gap; - in_gap = (npageslabs_huge == 0 && npageslabs_nonhuge == 0); - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } + bool is_gap = (s.npageslabs_huge == 0 && s.npageslabs_nonhuge == 0); col_size.size_val = sz_pind2sz(j); col_ind.size_val = j; - col_npageslabs_huge.size_val = npageslabs_huge; - col_nactive_huge.size_val = nactive_huge; - col_ndirty_huge.size_val = ndirty_huge; - col_npageslabs_nonhuge.size_val = npageslabs_nonhuge; - col_nactive_nonhuge.size_val = nactive_nonhuge; - col_ndirty_nonhuge.size_val = ndirty_nonhuge; - col_nretained_nonhuge.size_val = nretained_nonhuge; - if (!in_gap) { - emitter_table_row(emitter, &row); - } + col_npageslabs_huge.size_val = s.npageslabs_huge; + col_nactive_huge.size_val = s.nactive_huge; + col_ndirty_huge.size_val = s.ndirty_huge; + col_npageslabs_nonhuge.size_val = s.npageslabs_nonhuge; + col_nactive_nonhuge.size_val = s.nactive_nonhuge; + col_ndirty_nonhuge.size_val = s.ndirty_nonhuge; + col_nretained_nonhuge.size_val = s.nretained_nonhuge; + emitter_table_sparse_row(emitter, &row, is_gap); emitter_json_object_begin(emitter); emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size, - &npageslabs_huge); + &s.npageslabs_huge); emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter, "nactive_huge", emitter_type_size, &s.nactive_huge); emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); + emitter, "ndirty_huge", emitter_type_size, &s.ndirty_huge); emitter_json_kv(emitter, "npageslabs_nonhuge", - emitter_type_size, &npageslabs_nonhuge); + emitter_type_size, &s.npageslabs_nonhuge); emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size, - &nactive_nonhuge); + &s.nactive_nonhuge); emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size, - &ndirty_nonhuge); + &s.ndirty_nonhuge); emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); + &s.nretained_nonhuge); emitter_json_object_end(emitter); } emitter_json_array_end(emitter); /* End "nonfull_slabs" */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); } static void @@ -1250,6 +1444,11 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { static void stats_arena_mutexes_print( emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { + /* + * Mutex readers gather counters directly into emitter columns, coupling + * collection to the output representation. Emission remains a separate + * mutex_stats_emit() call. + */ emitter_row_t row; emitter_col_t col_name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; @@ -1278,90 +1477,57 @@ stats_arena_mutexes_print( emitter_json_object_end(emitter); /* End "mutexes". */ } -JEMALLOC_COLD static void -stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, - bool mutex, bool extents, bool hpa) { - char name[ARENA_NAME_LEN]; - char *namep = name; - unsigned nthreads; - const char *dss; - ssize_t dirty_decay_ms, muzzy_decay_ms; - size_t page, pactive, pdirty, pmuzzy, mapped, retained, pinned; - size_t base, internal, resident, metadata_edata, metadata_rtree, - metadata_thp, extent_avail; - uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; - uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; - size_t small_allocated; - uint64_t small_nmalloc, small_ndalloc, small_nrequests, small_nfills, - small_nflushes; - size_t large_allocated; - uint64_t large_nmalloc, large_ndalloc, large_nrequests, large_nfills, - large_nflushes; - size_t tcache_bytes, tcache_stashed_bytes, abandoned_vm; - uint64_t uptime; - - CTL_GET("arenas.page", &page, size_t); - if (i != MALLCTL_ARENAS_ALL && i != MALLCTL_ARENAS_DESTROYED) { - CTL_M1_GET("arena.0.name", i, (void *)&namep, const char *); +stats_emit_arena_basics(emitter_t *emitter, const stats_arena_basics_t *b) { + if (b->has_name) { + const char *namep = b->name; emitter_kv( emitter, "name", "name", emitter_type_string, &namep); } - - CTL_M2_GET("stats.arenas.0.nthreads", i, &nthreads, unsigned); emitter_kv(emitter, "nthreads", "assigned threads", - emitter_type_unsigned, &nthreads); - - CTL_M2_GET("stats.arenas.0.uptime", i, &uptime, uint64_t); - emitter_kv( - emitter, "uptime_ns", "uptime", emitter_type_uint64, &uptime); - - CTL_M2_GET("stats.arenas.0.dss", i, &dss, const char *); + emitter_type_unsigned, &b->nthreads); + emitter_kv(emitter, "uptime_ns", "uptime", emitter_type_uint64, + &b->uptime); emitter_kv(emitter, "dss", "dss allocation precedence", - emitter_type_string, &dss); + emitter_type_string, &b->dss); +} - CTL_M2_GET( - "stats.arenas.0.dirty_decay_ms", i, &dirty_decay_ms, ssize_t); - CTL_M2_GET( - "stats.arenas.0.muzzy_decay_ms", i, &muzzy_decay_ms, ssize_t); - CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t); - CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t); - CTL_M2_GET("stats.arenas.0.pmuzzy", i, &pmuzzy, size_t); - CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &dirty_npurge, uint64_t); - CTL_M2_GET( - "stats.arenas.0.dirty_nmadvise", i, &dirty_nmadvise, uint64_t); - CTL_M2_GET("stats.arenas.0.dirty_purged", i, &dirty_purged, uint64_t); - CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &muzzy_npurge, uint64_t); - CTL_M2_GET( - "stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise, uint64_t); - CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &muzzy_purged, uint64_t); +static uint64_t +stats_arena_basics_print(emitter_t *emitter, unsigned i) { + stats_arena_basics_t basics; + stats_gather_arena_basics(i, &basics); + stats_emit_arena_basics(emitter, &basics); + return basics.uptime; +} +static void +stats_emit_arena_decay(emitter_t *emitter, const stats_arena_decay_t *d) { emitter_row_t decay_row; emitter_row_init(&decay_row); /* JSON-style emission. */ emitter_json_kv( - emitter, "dirty_decay_ms", emitter_type_ssize, &dirty_decay_ms); + emitter, "dirty_decay_ms", emitter_type_ssize, &d->dirty_decay_ms); emitter_json_kv( - emitter, "muzzy_decay_ms", emitter_type_ssize, &muzzy_decay_ms); + emitter, "muzzy_decay_ms", emitter_type_ssize, &d->muzzy_decay_ms); - emitter_json_kv(emitter, "pactive", emitter_type_size, &pactive); - emitter_json_kv(emitter, "pdirty", emitter_type_size, &pdirty); - emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &pmuzzy); + emitter_json_kv(emitter, "pactive", emitter_type_size, &d->pactive); + emitter_json_kv(emitter, "pdirty", emitter_type_size, &d->pdirty); + emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &d->pmuzzy); emitter_json_kv( - emitter, "dirty_npurge", emitter_type_uint64, &dirty_npurge); + emitter, "dirty_npurge", emitter_type_uint64, &d->dirty_npurge); emitter_json_kv( - emitter, "dirty_nmadvise", emitter_type_uint64, &dirty_nmadvise); + emitter, "dirty_nmadvise", emitter_type_uint64, &d->dirty_nmadvise); emitter_json_kv( - emitter, "dirty_purged", emitter_type_uint64, &dirty_purged); + emitter, "dirty_purged", emitter_type_uint64, &d->dirty_purged); emitter_json_kv( - emitter, "muzzy_npurge", emitter_type_uint64, &muzzy_npurge); + emitter, "muzzy_npurge", emitter_type_uint64, &d->muzzy_npurge); emitter_json_kv( - emitter, "muzzy_nmadvise", emitter_type_uint64, &muzzy_nmadvise); + emitter, "muzzy_nmadvise", emitter_type_uint64, &d->muzzy_nmadvise); emitter_json_kv( - emitter, "muzzy_purged", emitter_type_uint64, &muzzy_purged); + emitter, "muzzy_purged", emitter_type_uint64, &d->muzzy_purged); /* Table-style emission. */ COL(decay_row, decay_type, right, 9, title); @@ -1388,54 +1554,65 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, /* Dirty row. */ col_decay_type.str_val = "dirty:"; - if (dirty_decay_ms >= 0) { + if (d->dirty_decay_ms >= 0) { col_decay_time.type = emitter_type_ssize; - col_decay_time.ssize_val = dirty_decay_ms; + col_decay_time.ssize_val = d->dirty_decay_ms; } else { col_decay_time.type = emitter_type_title; col_decay_time.str_val = "N/A"; } col_decay_npages.type = emitter_type_size; - col_decay_npages.size_val = pdirty; + col_decay_npages.size_val = d->pdirty; col_decay_sweeps.type = emitter_type_uint64; - col_decay_sweeps.uint64_val = dirty_npurge; + col_decay_sweeps.uint64_val = d->dirty_npurge; col_decay_madvises.type = emitter_type_uint64; - col_decay_madvises.uint64_val = dirty_nmadvise; + col_decay_madvises.uint64_val = d->dirty_nmadvise; col_decay_purged.type = emitter_type_uint64; - col_decay_purged.uint64_val = dirty_purged; + col_decay_purged.uint64_val = d->dirty_purged; emitter_table_row(emitter, &decay_row); /* Muzzy row. */ col_decay_type.str_val = "muzzy:"; - if (muzzy_decay_ms >= 0) { + if (d->muzzy_decay_ms >= 0) { col_decay_time.type = emitter_type_ssize; - col_decay_time.ssize_val = muzzy_decay_ms; + col_decay_time.ssize_val = d->muzzy_decay_ms; } else { col_decay_time.type = emitter_type_title; col_decay_time.str_val = "N/A"; } col_decay_npages.type = emitter_type_size; - col_decay_npages.size_val = pmuzzy; + col_decay_npages.size_val = d->pmuzzy; col_decay_sweeps.type = emitter_type_uint64; - col_decay_sweeps.uint64_val = muzzy_npurge; + col_decay_sweeps.uint64_val = d->muzzy_npurge; col_decay_madvises.type = emitter_type_uint64; - col_decay_madvises.uint64_val = muzzy_nmadvise; + col_decay_madvises.uint64_val = d->muzzy_nmadvise; col_decay_purged.type = emitter_type_uint64; - col_decay_purged.uint64_val = muzzy_purged; + col_decay_purged.uint64_val = d->muzzy_purged; emitter_table_row(emitter, &decay_row); +} - /* Small / large / total allocation counts. */ +static size_t +stats_arena_decay_print(emitter_t *emitter, unsigned i) { + stats_arena_decay_t decay; + stats_gather_arena_decay(i, &decay); + stats_emit_arena_decay(emitter, &decay); + return decay.pactive; +} + +static void +stats_emit_arena_alloc(emitter_t *emitter, const stats_arena_alloc_t *small, + const stats_arena_alloc_t *large, uint64_t uptime) { emitter_row_t alloc_count_row; emitter_row_init(&alloc_count_row); @@ -1478,70 +1655,78 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, col_count_nfills_ps.type = emitter_type_uint64; col_count_nflushes_ps.type = emitter_type_uint64; -#define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ - CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ - &small_or_large##_##name, valtype##_t); \ - emitter_json_kv( \ - emitter, #name, emitter_type_##valtype, &small_or_large##_##name); \ + /* + * JSON keys for the value columns; emitter_row() emits these as kvs. + * The (#/sec) rate columns and the title column stay table-only (no + * json_key). + */ + col_count_allocated.json_key = "allocated"; + col_count_nmalloc.json_key = "nmalloc"; + col_count_ndalloc.json_key = "ndalloc"; + col_count_nrequests.json_key = "nrequests"; + col_count_nfills.json_key = "nfills"; + col_count_nflushes.json_key = "nflushes"; + +#define EMIT_ALLOC_STAT(a, name, valtype) \ col_count_##name.type = emitter_type_##valtype; \ - col_count_##name.valtype##_val = small_or_large##_##name; + col_count_##name.valtype##_val = (a)->name; emitter_json_object_kv_begin(emitter, "small"); col_count_title.str_val = "small:"; - GET_AND_EMIT_ALLOC_STAT(small, allocated, size) - GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64) + EMIT_ALLOC_STAT(small, allocated, size) + EMIT_ALLOC_STAT(small, nmalloc, uint64) col_count_nmalloc_ps.uint64_val = rate_per_second( col_count_nmalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64) + EMIT_ALLOC_STAT(small, ndalloc, uint64) col_count_ndalloc_ps.uint64_val = rate_per_second( col_count_ndalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64) + EMIT_ALLOC_STAT(small, nrequests, uint64) col_count_nrequests_ps.uint64_val = rate_per_second( col_count_nrequests.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nfills, uint64) + EMIT_ALLOC_STAT(small, nfills, uint64) col_count_nfills_ps.uint64_val = rate_per_second( col_count_nfills.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nflushes, uint64) + EMIT_ALLOC_STAT(small, nflushes, uint64) col_count_nflushes_ps.uint64_val = rate_per_second( col_count_nflushes.uint64_val, uptime); - emitter_table_row(emitter, &alloc_count_row); + emitter_row(emitter, &alloc_count_row); emitter_json_object_end(emitter); /* Close "small". */ emitter_json_object_kv_begin(emitter, "large"); col_count_title.str_val = "large:"; - GET_AND_EMIT_ALLOC_STAT(large, allocated, size) - GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64) + EMIT_ALLOC_STAT(large, allocated, size) + EMIT_ALLOC_STAT(large, nmalloc, uint64) col_count_nmalloc_ps.uint64_val = rate_per_second( col_count_nmalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64) + EMIT_ALLOC_STAT(large, ndalloc, uint64) col_count_ndalloc_ps.uint64_val = rate_per_second( col_count_ndalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64) + EMIT_ALLOC_STAT(large, nrequests, uint64) col_count_nrequests_ps.uint64_val = rate_per_second( col_count_nrequests.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nfills, uint64) + EMIT_ALLOC_STAT(large, nfills, uint64) col_count_nfills_ps.uint64_val = rate_per_second( col_count_nfills.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nflushes, uint64) + EMIT_ALLOC_STAT(large, nflushes, uint64) col_count_nflushes_ps.uint64_val = rate_per_second( col_count_nflushes.uint64_val, uptime); - emitter_table_row(emitter, &alloc_count_row); + emitter_row(emitter, &alloc_count_row); emitter_json_object_end(emitter); /* Close "large". */ -#undef GET_AND_EMIT_ALLOC_STAT +#undef EMIT_ALLOC_STAT /* Aggregated small + large stats are emitter only in table mode. */ col_count_title.str_val = "total:"; - col_count_allocated.size_val = small_allocated + large_allocated; - col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; - col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; - col_count_nrequests.uint64_val = small_nrequests + large_nrequests; - col_count_nfills.uint64_val = small_nfills + large_nfills; - col_count_nflushes.uint64_val = small_nflushes + large_nflushes; + col_count_allocated.size_val = small->allocated + large->allocated; + col_count_nmalloc.uint64_val = small->nmalloc + large->nmalloc; + col_count_ndalloc.uint64_val = small->ndalloc + large->ndalloc; + col_count_nrequests.uint64_val = small->nrequests + large->nrequests; + col_count_nfills.uint64_val = small->nfills + large->nfills; + col_count_nflushes.uint64_val = small->nflushes + large->nflushes; col_count_nmalloc_ps.uint64_val = rate_per_second( col_count_nmalloc.uint64_val, uptime); col_count_ndalloc_ps.uint64_val = rate_per_second( @@ -1553,7 +1738,18 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, col_count_nflushes_ps.uint64_val = rate_per_second( col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); +} +static void +stats_arena_alloc_print(emitter_t *emitter, unsigned i, uint64_t uptime) { + stats_arena_alloc_t small_alloc, large_alloc; + stats_gather_arena_alloc(i, &small_alloc, &large_alloc); + stats_emit_arena_alloc(emitter, &small_alloc, &large_alloc, uptime); +} + +static void +stats_emit_arena_mem(emitter_t *emitter, const stats_arena_mem_t *mem, + size_t active_bytes) { emitter_row_t mem_count_row; emitter_row_init(&mem_count_row); @@ -1571,35 +1767,59 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, mem_count_val.type = emitter_type_title; mem_count_val.str_val = ""; - emitter_table_row(emitter, &mem_count_row); + /* Blank spacer row (both columns are empty titles; table only). */ + emitter_row(emitter, &mem_count_row); mem_count_val.type = emitter_type_size; - /* Active count in bytes is emitted only in table mode. */ + /* + * Active count in bytes is table only: mem_count_val carries no + * json_key, so emitter_row() skips it in JSON. + */ mem_count_title.str_val = "active:"; - mem_count_val.size_val = pactive * page; - emitter_table_row(emitter, &mem_count_row); + mem_count_val.size_val = active_bytes; + emitter_row(emitter, &mem_count_row); -#define GET_AND_EMIT_MEM_STAT(stat) \ - CTL_M2_GET("stats.arenas.0." #stat, i, &stat, size_t); \ - emitter_json_kv(emitter, #stat, emitter_type_size, &stat); \ + /* + * Each remaining stat is one row: the value column's json_key makes it a + * "": value pair in JSON and the aligned value in the table. + */ +#define EMIT_MEM_STAT(stat) \ mem_count_title.str_val = #stat ":"; \ - mem_count_val.size_val = stat; \ - emitter_table_row(emitter, &mem_count_row); + mem_count_val.json_key = #stat; \ + mem_count_val.size_val = mem->stat; \ + emitter_row(emitter, &mem_count_row); - GET_AND_EMIT_MEM_STAT(mapped) - GET_AND_EMIT_MEM_STAT(retained) - GET_AND_EMIT_MEM_STAT(pinned) - GET_AND_EMIT_MEM_STAT(base) - GET_AND_EMIT_MEM_STAT(internal) - GET_AND_EMIT_MEM_STAT(metadata_edata) - GET_AND_EMIT_MEM_STAT(metadata_rtree) - GET_AND_EMIT_MEM_STAT(metadata_thp) - GET_AND_EMIT_MEM_STAT(tcache_bytes) - GET_AND_EMIT_MEM_STAT(tcache_stashed_bytes) - GET_AND_EMIT_MEM_STAT(resident) - GET_AND_EMIT_MEM_STAT(abandoned_vm) - GET_AND_EMIT_MEM_STAT(extent_avail) -#undef GET_AND_EMIT_MEM_STAT + EMIT_MEM_STAT(mapped) + EMIT_MEM_STAT(retained) + EMIT_MEM_STAT(pinned) + EMIT_MEM_STAT(base) + EMIT_MEM_STAT(internal) + EMIT_MEM_STAT(metadata_edata) + EMIT_MEM_STAT(metadata_rtree) + EMIT_MEM_STAT(metadata_thp) + EMIT_MEM_STAT(tcache_bytes) + EMIT_MEM_STAT(tcache_stashed_bytes) + EMIT_MEM_STAT(resident) + EMIT_MEM_STAT(abandoned_vm) + EMIT_MEM_STAT(extent_avail) +#undef EMIT_MEM_STAT +} + +static void +stats_arena_mem_print(emitter_t *emitter, unsigned i, size_t pactive) { + stats_arena_mem_t m; + stats_gather_arena_mem(i, &m); + stats_emit_arena_mem(emitter, &m, pactive * m.page); +} + +JEMALLOC_COLD +static void +stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, + bool mutex, bool extents, bool hpa) { + uint64_t uptime = stats_arena_basics_print(emitter, i); + size_t pactive = stats_arena_decay_print(emitter, i); + stats_arena_alloc_print(emitter, i, uptime); + stats_arena_mem_print(emitter, i, pactive); if (opt_pac_sec_opts.nshards > 0) { stats_arena_pac_sec_print(emitter, i); @@ -1622,9 +1842,68 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, } } -JEMALLOC_COLD static void -stats_general_print(emitter_t *emitter) { +stats_general_version(emitter_t *emitter) { + const char *cpv; + + CTL_GET("version", &cpv, const char *); + emitter_kv(emitter, "version", "Version", emitter_type_string, &cpv); +} + +static void +stats_general_config(emitter_t *emitter) { + /* + * Most config entries are independent scalars that + * CONFIG_WRITE_BOOL reads and emits together. Keeping those operations + * local is clearer than introducing a gather struct for this flat list. + */ + bool bv; + + emitter_dict_begin(emitter, "config", "Build-time option settings"); + CONFIG_WRITE_BOOL(cache_oblivious); + CONFIG_WRITE_BOOL(debug); + CONFIG_WRITE_BOOL(fill); + CONFIG_WRITE_BOOL(infallible_new); + CONFIG_WRITE_BOOL(lazy_lock); + emitter_kv(emitter, "malloc_conf", "config.malloc_conf", + emitter_type_string, &config_malloc_conf); + + CONFIG_WRITE_BOOL(opt_safety_checks); + CONFIG_WRITE_BOOL(prof); + CONFIG_WRITE_BOOL(prof_libgcc); + CONFIG_WRITE_BOOL(prof_libunwind); + CONFIG_WRITE_BOOL(prof_frameptr); + CONFIG_WRITE_BOOL(stats); + CONFIG_WRITE_BOOL(utrace); + CONFIG_WRITE_BOOL(xmalloc); + emitter_dict_end(emitter); /* Close "config" dict. */ +} + +static void +stats_general_system(emitter_t *emitter) { + emitter_dict_begin(emitter, "system", "System configuration"); + + /* + * This shows system's THP mode detected at jemalloc's init time. + * jemalloc does not re-detect the mode even if it changes after + * jemalloc's init. It is assumed that system's THP mode is stable + * during the process's lifetime and a violation could lead to + * undefined behavior. + */ + const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode]; + emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string, + &thp_mode_name); + + emitter_dict_end(emitter); /* Close "system". */ +} + +static void +stats_general_opts(emitter_t *emitter) { + /* + * The OPT_WRITE_* macros conditionally read and emit most options. + * Options needing special formatting or availability checks stay + * explicit below. + */ const char *cpv; bool bv, bv2; unsigned uv; @@ -1643,82 +1922,6 @@ stats_general_print(emitter_t *emitter) { i64sz = sizeof(int64_t); u64sz = sizeof(uint64_t); - CTL_GET("version", &cpv, const char *); - emitter_kv(emitter, "version", "Version", emitter_type_string, &cpv); - - /* config. */ - emitter_dict_begin(emitter, "config", "Build-time option settings"); -#define CONFIG_WRITE_BOOL(name) \ - do { \ - CTL_GET("config." #name, &bv, bool); \ - emitter_kv( \ - emitter, #name, "config." #name, emitter_type_bool, &bv); \ - } while (0) - - CONFIG_WRITE_BOOL(cache_oblivious); - CONFIG_WRITE_BOOL(debug); - CONFIG_WRITE_BOOL(fill); - CONFIG_WRITE_BOOL(infallible_new); - CONFIG_WRITE_BOOL(lazy_lock); - emitter_kv(emitter, "malloc_conf", "config.malloc_conf", - emitter_type_string, &config_malloc_conf); - - CONFIG_WRITE_BOOL(opt_safety_checks); - CONFIG_WRITE_BOOL(prof); - CONFIG_WRITE_BOOL(prof_libgcc); - CONFIG_WRITE_BOOL(prof_libunwind); - CONFIG_WRITE_BOOL(prof_frameptr); - CONFIG_WRITE_BOOL(stats); - CONFIG_WRITE_BOOL(utrace); - CONFIG_WRITE_BOOL(xmalloc); -#undef CONFIG_WRITE_BOOL - emitter_dict_end(emitter); /* Close "config" dict. */ - - /* system. */ - emitter_dict_begin(emitter, "system", "System configuration"); - - /* - * This shows system's THP mode detected at jemalloc's init time. - * jemalloc does not re-detect the mode even if it changes after - * jemalloc's init. It is assumed that system's THP mode is stable - * during the process's lifetime and a violation could lead to - * undefined behavior. - */ - const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode]; - emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string, - &thp_mode_name); - - emitter_dict_end(emitter); /* Close "system". */ - - /* opt. */ -#define OPT_WRITE(name, var, size, emitter_type) \ - if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \ - emitter_kv(emitter, name, "opt." name, emitter_type, &var); \ - } - -#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname) \ - if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0 \ - && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) { \ - emitter_kv_note(emitter, name, "opt." name, emitter_type, \ - &var1, altname, emitter_type, &var2); \ - } - -#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool) -#define OPT_WRITE_BOOL_MUTABLE(name, altname) \ - OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname) - -#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned) - -#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64) -#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64) - -#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size) -#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize) -#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ - OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname) - -#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string) - emitter_dict_begin(emitter, "opt", "Run-time option settings"); /* @@ -1847,18 +2050,15 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("disable_large_size_classes") emitter_dict_end(emitter); /* Close "opt". */ +} -#undef OPT_WRITE -#undef OPT_WRITE_MUTABLE -#undef OPT_WRITE_BOOL -#undef OPT_WRITE_BOOL_MUTABLE -#undef OPT_WRITE_UNSIGNED -#undef OPT_WRITE_SSIZE_T -#undef OPT_WRITE_SSIZE_T_MUTABLE -#undef OPT_WRITE_CHAR_P - - /* prof. */ +static void +stats_general_prof(emitter_t *emitter) { if (config_prof) { + bool bv; + uint64_t u64v; + ssize_t ssv; + emitter_dict_begin(emitter, "prof", "Profiling settings"); CTL_GET("prof.thread_active_init", &bv, bool); @@ -1883,105 +2083,101 @@ stats_general_print(emitter_t *emitter) { emitter_dict_end(emitter); /* Close "prof". */ } +} + +static void +stats_general_bin_meta_print(emitter_t *emitter, unsigned nbins) { + /* + * Streaming JSON-only size-class metadata table (per-row gather+emit); + * see stats_arena_bins_print for the streaming rationale. + */ + emitter_json_array_kv_begin(emitter, "bin"); + size_t mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(mib, 0, "arenas.bin"); + for (unsigned i = 0; i < nbins; i++) { + stats_arena_bin_meta_t bm; + stats_gather_arena_bin_meta(mib, i, &bm); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "size", emitter_type_size, &bm.size); + emitter_json_kv(emitter, "nregs", emitter_type_uint32, &bm.nregs); + emitter_json_kv( + emitter, "slab_size", emitter_type_size, &bm.slab_size); + emitter_json_kv(emitter, "nshards", emitter_type_uint32, &bm.nshards); + emitter_json_object_end(emitter); + } + emitter_json_array_end(emitter); /* Close "bin". */ +} + +static void +stats_general_lextent_meta_print(emitter_t *emitter, unsigned nlextents) { + /* Streaming JSON-only metadata table; see stats_general_bin_meta_print. */ + emitter_json_array_kv_begin(emitter, "lextent"); + size_t mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(mib, 0, "arenas.lextent"); + for (unsigned i = 0; i < nlextents; i++) { + stats_arena_lextent_meta_t lm; + stats_gather_arena_lextent_meta(mib, i, &lm); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "size", emitter_type_size, &lm.size); + emitter_json_object_end(emitter); + } + emitter_json_array_end(emitter); /* Close "lextent". */ +} + +static void +stats_general_arenas_print(emitter_t *emitter, bool omit_size_class_meta) { + stats_arena_config_t cfg; + stats_gather_arena_config(&cfg); - /* arenas. */ /* * The json output sticks arena info into an "arenas" dict; the table * output puts them at the top-level. */ emitter_json_object_kv_begin(emitter, "arenas"); - CTL_GET("arenas.narenas", &uv, unsigned); - emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv); - - /* - * Decay settings are emitted only in json mode; in table mode, they're - * emitted as notes with the opt output, above. - */ - CTL_GET("arenas.dirty_decay_ms", &ssv, ssize_t); - emitter_json_kv(emitter, "dirty_decay_ms", emitter_type_ssize, &ssv); - - CTL_GET("arenas.muzzy_decay_ms", &ssv, ssize_t); - emitter_json_kv(emitter, "muzzy_decay_ms", emitter_type_ssize, &ssv); - - CTL_GET("arenas.quantum", &sv, size_t); - emitter_kv(emitter, "quantum", "Quantum size", emitter_type_size, &sv); - - CTL_GET("arenas.page", &sv, size_t); - emitter_kv(emitter, "page", "Page size", emitter_type_size, &sv); - - CTL_GET("arenas.hugepage", &sv, size_t); emitter_kv( - emitter, "hugepage", "Hugepage size", emitter_type_size, &sv); - - if (je_mallctl("arenas.tcache_max", (void *)&sv, &ssz, NULL, 0) == 0) { - emitter_kv(emitter, "tcache_max", - "Maximum thread-cached size class", emitter_type_size, &sv); - } - - unsigned arenas_nbins; - CTL_GET("arenas.nbins", &arenas_nbins, unsigned); - emitter_kv(emitter, "nbins", "Number of bin size classes", - emitter_type_unsigned, &arenas_nbins); - - unsigned arenas_nhbins; - CTL_GET("arenas.nhbins", &arenas_nhbins, unsigned); - emitter_kv(emitter, "nhbins", "Number of thread-cache bin size classes", - emitter_type_unsigned, &arenas_nhbins); + emitter, "narenas", "Arenas", emitter_type_unsigned, &cfg.narenas); /* - * We do enough mallctls in a loop that we actually want to omit them - * (not just omit the printing). + * Decay settings are emitted only in json mode; in table mode, they + * are emitted as notes with the opt output, above. */ - if (emitter_outputs_json(emitter)) { - emitter_json_array_kv_begin(emitter, "bin"); - size_t arenas_bin_mib[CTL_MAX_DEPTH]; - CTL_LEAF_PREPARE(arenas_bin_mib, 0, "arenas.bin"); - for (unsigned i = 0; i < arenas_nbins; i++) { - arenas_bin_mib[2] = i; - emitter_json_object_begin(emitter); + emitter_json_kv( + emitter, "dirty_decay_ms", emitter_type_ssize, &cfg.dirty_decay_ms); + emitter_json_kv( + emitter, "muzzy_decay_ms", emitter_type_ssize, &cfg.muzzy_decay_ms); - CTL_LEAF(arenas_bin_mib, 3, "size", &sv, size_t); - emitter_json_kv( - emitter, "size", emitter_type_size, &sv); + emitter_kv(emitter, "quantum", "Quantum size", emitter_type_size, + &cfg.quantum); + emitter_kv(emitter, "page", "Page size", emitter_type_size, &cfg.page); + emitter_kv(emitter, "hugepage", "Hugepage size", emitter_type_size, + &cfg.hugepage); - CTL_LEAF(arenas_bin_mib, 3, "nregs", &u32v, uint32_t); - emitter_json_kv( - emitter, "nregs", emitter_type_uint32, &u32v); - - CTL_LEAF(arenas_bin_mib, 3, "slab_size", &sv, size_t); - emitter_json_kv( - emitter, "slab_size", emitter_type_size, &sv); - - CTL_LEAF(arenas_bin_mib, 3, "nshards", &u32v, uint32_t); - emitter_json_kv( - emitter, "nshards", emitter_type_uint32, &u32v); - - emitter_json_object_end(emitter); - } - emitter_json_array_end(emitter); /* Close "bin". */ + if (cfg.have_tcache_max) { + emitter_kv(emitter, "tcache_max", + "Maximum thread-cached size class", emitter_type_size, + &cfg.tcache_max); + } + + emitter_kv(emitter, "nbins", "Number of bin size classes", + emitter_type_unsigned, &cfg.nbins); + emitter_kv(emitter, "nhbins", "Number of thread-cache bin size classes", + emitter_type_unsigned, &cfg.nhbins); + + /* + * Per-size-class geometry is omitted from table output. It is present + * in JSON whenever general output is enabled; when omitted, it is not + * gathered either. + */ + if (!omit_size_class_meta) { + stats_general_bin_meta_print(emitter, cfg.nbins); } - unsigned nlextents; - CTL_GET("arenas.nlextents", &nlextents, unsigned); emitter_kv(emitter, "nlextents", "Number of large size classes", - emitter_type_unsigned, &nlextents); + emitter_type_unsigned, &cfg.nlextents); - if (emitter_outputs_json(emitter)) { - emitter_json_array_kv_begin(emitter, "lextent"); - size_t arenas_lextent_mib[CTL_MAX_DEPTH]; - CTL_LEAF_PREPARE(arenas_lextent_mib, 0, "arenas.lextent"); - for (unsigned i = 0; i < nlextents; i++) { - arenas_lextent_mib[2] = i; - emitter_json_object_begin(emitter); - - CTL_LEAF(arenas_lextent_mib, 3, "size", &sv, size_t); - emitter_json_kv( - emitter, "size", emitter_type_size, &sv); - - emitter_json_object_end(emitter); - } - emitter_json_array_end(emitter); /* Close "lextent". */ + if (!omit_size_class_meta) { + stats_general_lextent_meta_print(emitter, cfg.nlextents); } emitter_json_object_end(emitter); /* Close "arenas" */ @@ -1989,258 +2185,166 @@ stats_general_print(emitter_t *emitter) { JEMALLOC_COLD static void -stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, - bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) { - /* - * These should be deleted. We keep them around for a while, to aid in - * the transition to the emitter code. - */ - size_t allocated, active, metadata, metadata_edata, metadata_rtree, - metadata_thp, resident, mapped, retained, pinned; - size_t num_background_threads; - size_t zero_reallocs; - uint64_t background_thread_num_runs, background_thread_run_interval; - - CTL_GET("stats.allocated", &allocated, size_t); - CTL_GET("stats.active", &active, size_t); - CTL_GET("stats.metadata", &metadata, size_t); - CTL_GET("stats.metadata_edata", &metadata_edata, size_t); - CTL_GET("stats.metadata_rtree", &metadata_rtree, size_t); - CTL_GET("stats.metadata_thp", &metadata_thp, size_t); - CTL_GET("stats.resident", &resident, size_t); - CTL_GET("stats.mapped", &mapped, size_t); - CTL_GET("stats.retained", &retained, size_t); - CTL_GET("stats.pinned", &pinned, size_t); - - CTL_GET("stats.zero_reallocs", &zero_reallocs, size_t); - - if (have_background_thread) { - CTL_GET("stats.background_thread.num_threads", - &num_background_threads, size_t); - CTL_GET("stats.background_thread.num_runs", - &background_thread_num_runs, uint64_t); - CTL_GET("stats.background_thread.run_interval", - &background_thread_run_interval, uint64_t); - } else { - num_background_threads = 0; - background_thread_num_runs = 0; - background_thread_run_interval = 0; - } +stats_general_print(emitter_t *emitter, bool omit_size_class_meta) { + stats_general_version(emitter); + stats_general_config(emitter); + stats_general_system(emitter); + stats_general_opts(emitter); + stats_general_prof(emitter); + stats_general_arenas_print(emitter, omit_size_class_meta); +} +static void +stats_emit_global(emitter_t *emitter, const stats_global_t *g) { /* Generic global stats. */ - emitter_json_object_kv_begin(emitter, "stats"); - emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated); - emitter_json_kv(emitter, "active", emitter_type_size, &active); - emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata); + emitter_json_kv(emitter, "allocated", emitter_type_size, &g->allocated); + emitter_json_kv(emitter, "active", emitter_type_size, &g->active); + emitter_json_kv(emitter, "metadata", emitter_type_size, &g->metadata); emitter_json_kv( - emitter, "metadata_edata", emitter_type_size, &metadata_edata); + emitter, "metadata_edata", emitter_type_size, &g->metadata_edata); emitter_json_kv( - emitter, "metadata_rtree", emitter_type_size, &metadata_rtree); + emitter, "metadata_rtree", emitter_type_size, &g->metadata_rtree); emitter_json_kv( - emitter, "metadata_thp", emitter_type_size, &metadata_thp); - emitter_json_kv(emitter, "resident", emitter_type_size, &resident); - emitter_json_kv(emitter, "mapped", emitter_type_size, &mapped); - emitter_json_kv(emitter, "retained", emitter_type_size, &retained); - emitter_json_kv(emitter, "pinned", emitter_type_size, &pinned); + emitter, "metadata_thp", emitter_type_size, &g->metadata_thp); + emitter_json_kv(emitter, "resident", emitter_type_size, &g->resident); + emitter_json_kv(emitter, "mapped", emitter_type_size, &g->mapped); + emitter_json_kv(emitter, "retained", emitter_type_size, &g->retained); + emitter_json_kv(emitter, "pinned", emitter_type_size, &g->pinned); emitter_json_kv( - emitter, "zero_reallocs", emitter_type_size, &zero_reallocs); + emitter, "zero_reallocs", emitter_type_size, &g->zero_reallocs); emitter_table_printf(emitter, "Allocated: %zu, active: %zu, " "metadata: %zu (n_thp %zu, edata %zu, rtree %zu), resident: %zu, " "mapped: %zu, retained: %zu, pinned: %zu\n", - allocated, active, metadata, metadata_thp, metadata_edata, - metadata_rtree, resident, mapped, retained, pinned); + g->allocated, g->active, g->metadata, g->metadata_thp, + g->metadata_edata, g->metadata_rtree, g->resident, g->mapped, + g->retained, g->pinned); /* Strange behaviors */ emitter_table_printf(emitter, - "Count of realloc(non-null-ptr, 0) calls: %zu\n", zero_reallocs); + "Count of realloc(non-null-ptr, 0) calls: %zu\n", g->zero_reallocs); /* Background thread stats. */ emitter_json_object_kv_begin(emitter, "background_thread"); - emitter_json_kv( - emitter, "num_threads", emitter_type_size, &num_background_threads); + emitter_json_kv(emitter, "num_threads", emitter_type_size, + &g->num_background_threads); emitter_json_kv(emitter, "num_runs", emitter_type_uint64, - &background_thread_num_runs); + &g->background_thread_num_runs); emitter_json_kv(emitter, "run_interval", emitter_type_uint64, - &background_thread_run_interval); + &g->background_thread_run_interval); emitter_json_object_end(emitter); /* Close "background_thread". */ emitter_table_printf(emitter, "Background threads: %zu, " "num_runs: %" FMTu64 ", run_interval: %" FMTu64 " ns\n", - num_background_threads, background_thread_num_runs, - background_thread_run_interval); - - if (mutex) { - emitter_row_t row; - emitter_col_t name; - emitter_col_t col64[mutex_prof_num_uint64_t_counters]; - emitter_col_t col32[mutex_prof_num_uint32_t_counters]; - uint64_t uptime; - - emitter_row_init(&row); - mutex_stats_init_cols(&row, "", &name, col64, col32); - - emitter_table_row(emitter, &row); - emitter_json_object_kv_begin(emitter, "mutexes"); - - CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); - - size_t stats_mutexes_mib[CTL_MAX_DEPTH]; - CTL_LEAF_PREPARE(stats_mutexes_mib, 0, "stats.mutexes"); - for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { - mutex_stats_read_global(stats_mutexes_mib, 2, - global_mutex_names[i], &name, col64, col32, uptime); - emitter_json_object_kv_begin( - emitter, global_mutex_names[i]); - mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_object_end(emitter); - } - - emitter_json_object_end(emitter); /* Close "mutexes". */ - } - - emitter_json_object_end(emitter); /* Close "stats". */ - - if (merged || destroyed || unmerged) { - unsigned narenas; - - emitter_json_object_kv_begin(emitter, "stats.arenas"); - - CTL_GET("arenas.narenas", &narenas, unsigned); - size_t mib[3]; - size_t miblen = sizeof(mib) / sizeof(size_t); - size_t sz; - VARIABLE_ARRAY_UNSAFE(bool, initialized, narenas); - bool destroyed_initialized; - unsigned i, ninitialized; - - xmallctlnametomib("arena.0.initialized", mib, &miblen); - for (i = ninitialized = 0; i < narenas; i++) { - mib[1] = i; - sz = sizeof(bool); - xmallctlbymib( - mib, miblen, &initialized[i], &sz, NULL, 0); - if (initialized[i]) { - ninitialized++; - } - } - mib[1] = MALLCTL_ARENAS_DESTROYED; - sz = sizeof(bool); - xmallctlbymib( - mib, miblen, &destroyed_initialized, &sz, NULL, 0); - - /* Merged stats. */ - if (merged && (ninitialized > 1 || !unmerged)) { - /* Print merged arena stats. */ - emitter_table_printf(emitter, "Merged arenas stats:\n"); - emitter_json_object_kv_begin(emitter, "merged"); - stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins, - large, mutex, extents, hpa); - emitter_json_object_end(emitter); /* Close "merged". */ - } - - /* Destroyed stats. */ - if (destroyed_initialized && destroyed) { - /* Print destroyed arena stats. */ - emitter_table_printf( - emitter, "Destroyed arenas stats:\n"); - emitter_json_object_kv_begin(emitter, "destroyed"); - stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, - bins, large, mutex, extents, hpa); - emitter_json_object_end( - emitter); /* Close "destroyed". */ - } - - /* Unmerged stats. */ - if (unmerged) { - for (i = 0; i < narenas; i++) { - if (initialized[i]) { - char arena_ind_str[20]; - malloc_snprintf(arena_ind_str, - sizeof(arena_ind_str), "%u", i); - emitter_json_object_kv_begin( - emitter, arena_ind_str); - emitter_table_printf(emitter, - "arenas[%s]:\n", arena_ind_str); - stats_arena_print(emitter, i, bins, - large, mutex, extents, hpa); - /* Close "". */ - emitter_json_object_end(emitter); - } - } - } - emitter_json_object_end(emitter); /* Close "stats.arenas". */ - } + g->num_background_threads, g->background_thread_num_runs, + g->background_thread_run_interval); } -void -stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { - int err; - uint64_t epoch; - size_t u64sz; -#define OPTION(o, v, d, s) bool v = d; - STATS_PRINT_OPTIONS -#undef OPTION +static void +stats_global_mutexes_print(emitter_t *emitter) { + /* Counters are gathered into emitter columns; see the arena variant. */ + emitter_row_t row; + emitter_col_t name; + emitter_col_t col64[mutex_prof_num_uint64_t_counters]; + emitter_col_t col32[mutex_prof_num_uint32_t_counters]; + uint64_t uptime; - /* - * Refresh stats, in case mallctl() was called by the application. - * - * Check for OOM here, since refreshing the ctl cache can trigger - * allocation. In practice, none of the subsequent mallctl()-related - * calls in this function will cause OOM if this one succeeds. - * */ - epoch = 1; - u64sz = sizeof(uint64_t); - err = je_mallctl( - "epoch", (void *)&epoch, &u64sz, (void *)&epoch, sizeof(uint64_t)); - if (err != 0) { - if (err == EAGAIN) { - malloc_write( - ": Memory allocation failure in " - "mallctl(\"epoch\", ...)\n"); - return; - } - malloc_write( - ": Failure in mallctl(\"epoch\", " - "...)\n"); - abort(); + emitter_row_init(&row); + mutex_stats_init_cols(&row, "", &name, col64, col32); + + emitter_table_row(emitter, &row); + emitter_json_object_kv_begin(emitter, "mutexes"); + + CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); + + size_t stats_mutexes_mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(stats_mutexes_mib, 0, "stats.mutexes"); + for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { + mutex_stats_read_global(stats_mutexes_mib, 2, + global_mutex_names[i], &name, col64, col32, uptime); + emitter_json_object_kv_begin(emitter, global_mutex_names[i]); + mutex_stats_emit(emitter, &row, col64, col32); + emitter_json_object_end(emitter); } - if (opts != NULL) { - for (unsigned i = 0; opts[i] != '\0'; i++) { - switch (opts[i]) { -#define OPTION(o, v, d, s) \ - case o: \ - v = s; \ - break; - STATS_PRINT_OPTIONS -#undef OPTION - default:; + emitter_json_object_end(emitter); /* Close "mutexes". */ +} + +static void +stats_print_globals(emitter_t *emitter, bool mutex) { + stats_global_t g; + stats_gather_global(&g); + + emitter_json_object_kv_begin(emitter, "stats"); + stats_emit_global(emitter, &g); + if (mutex) { + stats_global_mutexes_print(emitter); + } + emitter_json_object_end(emitter); /* Close "stats". */ +} + +static void +stats_print_one_arena(emitter_t *emitter, unsigned arena_ind, + const char *json_key, const char *table_header, bool bins, bool large, + bool mutex, bool extents, bool hpa) { + emitter_json_object_kv_begin(emitter, json_key); + emitter_table_printf(emitter, "%s", table_header); + stats_arena_print(emitter, arena_ind, bins, large, mutex, extents, hpa); + emitter_json_object_end(emitter); +} + +JEMALLOC_COLD +static void +stats_print_runtime_stats(emitter_t *emitter, bool merged, bool destroyed, + bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) { + stats_print_globals(emitter, mutex); + + if (!merged && !destroyed && !unmerged) { + return; + } + + unsigned narenas; + CTL_GET("arenas.narenas", &narenas, unsigned); + VARIABLE_ARRAY_UNSAFE(bool, initialized, narenas); + bool destroyed_initialized; + unsigned ninitialized = stats_gather_arenas_initialized(narenas, + initialized, &destroyed_initialized); + + emitter_json_object_kv_begin(emitter, "stats.arenas"); + + /* Merged stats. */ + if (merged && (ninitialized > 1 || !unmerged)) { + stats_print_one_arena(emitter, MALLCTL_ARENAS_ALL, "merged", + "Merged arenas stats:\n", bins, large, mutex, extents, hpa); + } + + /* Destroyed stats. */ + if (destroyed_initialized && destroyed) { + stats_print_one_arena(emitter, MALLCTL_ARENAS_DESTROYED, + "destroyed", "Destroyed arenas stats:\n", bins, large, mutex, + extents, hpa); + } + + /* Unmerged stats. */ + if (unmerged) { + for (unsigned i = 0; i < narenas; i++) { + if (!initialized[i]) { + continue; } + char arena_ind_str[20]; + malloc_snprintf(arena_ind_str, sizeof(arena_ind_str), + "%u", i); + char header[32]; + malloc_snprintf(header, sizeof(header), + "arenas[%s]:\n", arena_ind_str); + stats_print_one_arena(emitter, i, arena_ind_str, header, + bins, large, mutex, extents, hpa); } } - emitter_t emitter; - emitter_init(&emitter, - json ? emitter_output_json_compact : emitter_output_table, write_cb, - cbopaque); - emitter_begin(&emitter); - emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); - emitter_json_object_kv_begin(&emitter, "jemalloc"); - - if (general) { - stats_general_print(&emitter); - } - if (config_stats) { - stats_print_helper(&emitter, merged, destroyed, unmerged, bins, - large, mutex, extents, hpa); - } - - emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ - emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); - emitter_end(&emitter); + emitter_json_object_end(emitter); /* Close "stats.arenas". */ } static uint64_t diff --git a/test/unit/emitter.c b/test/unit/emitter.c index dc53b9eb..f209b773 100644 --- a/test/unit/emitter.c +++ b/test/unit/emitter.c @@ -498,13 +498,13 @@ emit_table_row(emitter_t *emitter) { emitter_begin(emitter); emitter_row_t row; emitter_col_t abc = { - emitter_justify_left, 10, emitter_type_title, {0}, {0, 0}}; + emitter_justify_left, 10, emitter_type_title}; abc.str_val = "ABC title"; emitter_col_t def = { - emitter_justify_right, 15, emitter_type_title, {0}, {0, 0}}; + emitter_justify_right, 15, emitter_type_title}; def.str_val = "DEF title"; emitter_col_t ghi = { - emitter_justify_right, 5, emitter_type_title, {0}, {0, 0}}; + emitter_justify_right, 5, emitter_type_title}; ghi.str_val = "GHI"; emitter_row_init(&row); @@ -548,6 +548,126 @@ static const char *table_row_table = "789 false 1011\n" "\"a string\" false ghi\n"; +/* + * emitter_row(): columns that carry a json_key render both to the aligned + * table (all columns) and, in JSON mode, to "json_key": value pairs in column + * order; columns without a json_key are table-only. + */ +static void +emit_row(emitter_t *emitter) { + emitter_begin(emitter); + + emitter_row_t row; + emitter_row_init(&row); + + /* Table-only column (no json_key set). */ + emitter_col_t col_label; + emitter_col_init(&col_label, &row); + col_label.justify = emitter_justify_left; + col_label.width = 6; + col_label.type = emitter_type_title; + col_label.str_val = "row:"; + + /* Dual columns: rendered in both table and JSON. */ + emitter_col_t col_x; + emitter_col_init(&col_x, &row); + col_x.justify = emitter_justify_right; + col_x.width = 6; + col_x.type = emitter_type_unsigned; + col_x.unsigned_val = 10; + col_x.json_key = "x"; + + emitter_col_t col_y; + emitter_col_init(&col_y, &row); + col_y.justify = emitter_justify_right; + col_y.width = 6; + col_y.type = emitter_type_unsigned; + col_y.unsigned_val = 20; + col_y.json_key = "y"; + + emitter_json_object_kv_begin(emitter, "obj"); + emitter_row(emitter, &row); + emitter_json_object_end(emitter); + + emitter_end(emitter); +} + +static const char *row_json = + "{\n" + "\t\"obj\": {\n" + "\t\t\"x\": 10,\n" + "\t\t\"y\": 20\n" + "\t}\n" + "}\n"; +static const char *row_json_compact = "{\"obj\":{\"x\":10,\"y\":20}}"; +static const char *row_table = "row: 10 20\n"; + +/* + * emitter_sparse_row(): the table side collapses runs of all-zero ("gap") rows + * to a single "---" separator (on a gap->non-gap transition and at a trailing + * gap), while the JSON side still emits every row's json-keyed columns. + */ +static void +emit_sparse_row(emitter_t *emitter) { + emitter_begin(emitter); + + emitter_row_t row; + emitter_row_init(&row); + emitter_col_t col_v; + emitter_col_init(&col_v, &row); + col_v.justify = emitter_justify_right; + col_v.width = 6; + col_v.type = emitter_type_unsigned; + col_v.json_key = "v"; + + unsigned vals[] = {1, 0, 0, 2, 0, 0}; + bool gap[] = {false, true, true, false, true, true}; + + emitter_json_array_kv_begin(emitter, "arr"); + emitter_table_sparse_begin(emitter); + for (unsigned i = 0; i < sizeof(vals) / sizeof(vals[0]); i++) { + col_v.unsigned_val = vals[i]; + emitter_json_object_begin(emitter); + emitter_sparse_row(emitter, &row, gap[i]); + emitter_json_object_end(emitter); + } + emitter_table_sparse_end(emitter); + emitter_json_array_end(emitter); + + emitter_end(emitter); +} + +static const char *sparse_row_json = + "{\n" + "\t\"arr\": [\n" + "\t\t{\n" + "\t\t\t\"v\": 1\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 2\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t}\n" + "\t]\n" + "}\n"; +static const char *sparse_row_json_compact = + "{\"arr\":[{\"v\":1},{\"v\":0},{\"v\":0},{\"v\":2},{\"v\":0},{\"v\":0}]}"; +static const char *sparse_row_table = + " 1\n" + " ---\n" + " 2\n" + " ---\n"; + #define GENERATE_TEST(feature) \ TEST_BEGIN(test_##feature) { \ expect_emit_output(emit_##feature, feature##_json, \ @@ -563,10 +683,12 @@ GENERATE_TEST(modal) GENERATE_TEST(json_array) GENERATE_TEST(json_nested_array) GENERATE_TEST(table_row) +GENERATE_TEST(row) +GENERATE_TEST(sparse_row) int main(void) { return test_no_reentrancy(test_dict, test_table_printf, test_nested_dict, test_types, test_modal, test_json_array, - test_json_nested_array, test_table_row); + test_json_nested_array, test_table_row, test_row, test_sparse_row); } From c3700194f03adbd6fc406d64769402c8452d105f Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Thu, 16 Jul 2026 05:39:01 -0700 Subject: [PATCH 3/9] Format change: HPA shard counters from prose to structured rows Table-output change: emit the HPA shard counters as structured key:value rows keyed by the JSON field names, instead of the previous free-form prose. JSON output is byte-identical; only the human-readable table changes. --- src/stats.c | 93 +++++++++++++++++++++++------------------------------ 1 file changed, 41 insertions(+), 52 deletions(-) diff --git a/src/stats.c b/src/stats.c index c09b49d4..9e9188a7 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1178,60 +1178,49 @@ stats_arena_pac_sec_print(emitter_t *emitter, unsigned i) { static void stats_emit_arena_hpa_counters(emitter_t *emitter, const stats_arena_hpa_counters_t *c, uint64_t uptime) { - emitter_table_printf(emitter, - "HPA shard stats:\n" - " Pageslabs: %zu (%zu huge, %zu nonhuge)\n" - " Active pages: %zu (%zu huge, %zu nonhuge)\n" - " Dirty pages: %zu (%zu huge, %zu nonhuge)\n" - " Retained pages: %zu\n" - " Purge passes: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Purges: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Hugeifies: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Hugify failures: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n", - c->npageslabs, c->npageslabs_huge, c->npageslabs_nonhuge, c->nactive, - c->nactive_huge, c->nactive_nonhuge, c->ndirty, c->ndirty_huge, - c->ndirty_nonhuge, c->nretained_nonhuge, c->npurge_passes, - rate_per_second(c->npurge_passes, uptime), c->npurges, - rate_per_second(c->npurges, uptime), c->nhugifies, - rate_per_second(c->nhugifies, uptime), c->nhugify_failures, - rate_per_second(c->nhugify_failures, uptime), c->ndehugifies, - rate_per_second(c->ndehugifies, uptime)); + /* Merged pageslab / page counts (broken down by huginess below). */ + emitter_kv(emitter, "npageslabs", "npageslabs", emitter_type_size, + &c->npageslabs); + emitter_kv(emitter, "nactive", "nactive", emitter_type_size, + &c->nactive); + emitter_kv(emitter, "ndirty", "ndirty", emitter_type_size, &c->ndirty); - emitter_json_kv(emitter, "npageslabs", emitter_type_size, &c->npageslabs); - emitter_json_kv(emitter, "nactive", emitter_type_size, &c->nactive); - emitter_json_kv(emitter, "ndirty", emitter_type_size, &c->ndirty); + uint64_t npurge_passes_ps = rate_per_second(c->npurge_passes, uptime); + uint64_t npurges_ps = rate_per_second(c->npurges, uptime); + uint64_t nhugifies_ps = rate_per_second(c->nhugifies, uptime); + uint64_t nhugify_failures_ps = + rate_per_second(c->nhugify_failures, uptime); + uint64_t ndehugifies_ps = rate_per_second(c->ndehugifies, uptime); + emitter_kv_note(emitter, "npurge_passes", "npurge_passes", + emitter_type_uint64, &c->npurge_passes, "per_sec", + emitter_type_uint64, &npurge_passes_ps); + emitter_kv_note(emitter, "npurges", "npurges", emitter_type_uint64, + &c->npurges, "per_sec", emitter_type_uint64, &npurges_ps); + emitter_kv_note(emitter, "nhugifies", "nhugifies", emitter_type_uint64, + &c->nhugifies, "per_sec", emitter_type_uint64, &nhugifies_ps); + emitter_kv_note(emitter, "nhugify_failures", "nhugify_failures", + emitter_type_uint64, &c->nhugify_failures, "per_sec", + emitter_type_uint64, &nhugify_failures_ps); + emitter_kv_note(emitter, "ndehugifies", "ndehugifies", + emitter_type_uint64, &c->ndehugifies, "per_sec", + emitter_type_uint64, &ndehugifies_ps); - emitter_json_kv( - emitter, "npurge_passes", emitter_type_uint64, &c->npurge_passes); - emitter_json_kv(emitter, "npurges", emitter_type_uint64, &c->npurges); - emitter_json_kv(emitter, "nhugifies", emitter_type_uint64, &c->nhugifies); - emitter_json_kv(emitter, "nhugify_failures", emitter_type_uint64, - &c->nhugify_failures); - emitter_json_kv( - emitter, "ndehugifies", emitter_type_uint64, &c->ndehugifies); - - emitter_json_object_kv_begin(emitter, "slabs"); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &c->npageslabs_nonhuge); - emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &c->nactive_nonhuge); - emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &c->ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &c->nretained_nonhuge); - - emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &c->npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &c->nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &c->ndirty_huge); - emitter_json_object_end(emitter); /* End "slabs" */ + emitter_dict_begin(emitter, "slabs", "slabs"); + emitter_kv(emitter, "npageslabs_nonhuge", "npageslabs_nonhuge", + emitter_type_size, &c->npageslabs_nonhuge); + emitter_kv(emitter, "nactive_nonhuge", "nactive_nonhuge", + emitter_type_size, &c->nactive_nonhuge); + emitter_kv(emitter, "ndirty_nonhuge", "ndirty_nonhuge", + emitter_type_size, &c->ndirty_nonhuge); + emitter_kv(emitter, "nretained_nonhuge", "nretained_nonhuge", + emitter_type_size, &c->nretained_nonhuge); + emitter_kv(emitter, "npageslabs_huge", "npageslabs_huge", + emitter_type_size, &c->npageslabs_huge); + emitter_kv(emitter, "nactive_huge", "nactive_huge", emitter_type_size, + &c->nactive_huge); + emitter_kv(emitter, "ndirty_huge", "ndirty_huge", emitter_type_size, + &c->ndirty_huge); + emitter_dict_end(emitter); } static void From 51ff051a02887ac277d88d7d14b15659c5e79c4f Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Thu, 16 Jul 2026 05:43:58 -0700 Subject: [PATCH 4/9] Format change: fold full/empty pageslabs into the nonfull table Table-output change: fold the HPA "In full slabs:" / "In empty slabs:" prose blocks into the per-size-class table as two symbolic rows (size "full"/"empty") and retitle it "pageslabs:". JSON is byte-identical; only the human-readable table changes. --- src/stats.c | 82 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 29 deletions(-) diff --git a/src/stats.c b/src/stats.c index 9e9188a7..1bf0a4da 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1313,18 +1313,8 @@ stats_arena_hpa_shard_counters_print( } static void -stats_emit_arena_hpa_slab(emitter_t *emitter, const stats_arena_hpa_slab_t *s, - const char *label, const char *json_key) { - emitter_table_printf(emitter, - " In %s slabs:\n" - " npageslabs: %zu huge, %zu nonhuge\n" - " nactive: %zu huge, %zu nonhuge \n" - " ndirty: %zu huge, %zu nonhuge \n" - " nretained: 0 huge, %zu nonhuge \n", - label, s->npageslabs_huge, s->npageslabs_nonhuge, s->nactive_huge, - s->nactive_nonhuge, s->ndirty_huge, s->ndirty_nonhuge, - s->nretained_nonhuge); - +stats_emit_arena_hpa_slab_json(emitter_t *emitter, + const stats_arena_hpa_slab_t *s, const char *json_key) { emitter_json_object_kv_begin(emitter, json_key); emitter_json_kv( emitter, "npageslabs_huge", emitter_type_size, &s->npageslabs_huge); @@ -1340,7 +1330,23 @@ stats_emit_arena_hpa_slab(emitter_t *emitter, const stats_arena_hpa_slab_t *s, emitter, "ndirty_nonhuge", emitter_type_size, &s->ndirty_nonhuge); emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, &s->nretained_nonhuge); - emitter_json_object_end(emitter); /* End "full_slabs"/"empty_slabs" */ + emitter_json_object_end(emitter); +} + +/* Fill the shared count columns of one pageslab row from s. */ +static void +stats_hpa_slab_cols_set(const stats_arena_hpa_slab_t *s, + emitter_col_t *npageslabs_huge, emitter_col_t *nactive_huge, + emitter_col_t *ndirty_huge, emitter_col_t *npageslabs_nonhuge, + emitter_col_t *nactive_nonhuge, emitter_col_t *ndirty_nonhuge, + emitter_col_t *nretained_nonhuge) { + npageslabs_huge->size_val = s->npageslabs_huge; + nactive_huge->size_val = s->nactive_huge; + ndirty_huge->size_val = s->ndirty_huge; + npageslabs_nonhuge->size_val = s->npageslabs_nonhuge; + nactive_nonhuge->size_val = s->nactive_nonhuge; + ndirty_nonhuge->size_val = s->ndirty_nonhuge; + nretained_nonhuge->size_val = s->nretained_nonhuge; } static void @@ -1350,20 +1356,6 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_row_t row; emitter_row_init(&row); - stats_arena_hpa_slab_t sfull; - stats_gather_arena_hpa_slab(i, "full_slabs", &sfull); - stats_emit_arena_hpa_slab(emitter, &sfull, "full", "full_slabs"); - - stats_arena_hpa_slab_t sempty; - stats_gather_arena_hpa_slab(i, "empty_slabs", &sempty); - stats_emit_arena_hpa_slab(emitter, &sempty, "empty", "empty_slabs"); - - /* - * Nonfull slabs: a per-size-class streaming table (gather one row, - * emit one row, collapsing all-zero-row runs to "---"), so gather and - * emit are necessarily interleaved -- like the bins/lextents/extents - * tables. - */ COL_HDR(row, size, NULL, right, 20, size) COL_HDR(row, ind, NULL, right, 4, unsigned) COL_HDR(row, npageslabs_huge, NULL, right, 16, size) @@ -1374,13 +1366,45 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { COL_HDR(row, ndirty_nonhuge, NULL, right, 20, size) COL_HDR(row, nretained_nonhuge, NULL, right, 20, size) + emitter_table_printf(emitter, "pageslabs:\n"); + emitter_table_row(emitter, &header_row); + + /* + * Full and empty pageslabs are shown as two symbolic rows (size = + * "full"/"empty", no size-class index) at the top of the table; their + * JSON stays in the separate full_slabs / empty_slabs objects. + */ + col_ind.type = emitter_type_title; + col_ind.str_val = "-"; + col_size.type = emitter_type_title; + + stats_arena_hpa_slab_t sfull; + stats_gather_arena_hpa_slab(i, "full_slabs", &sfull); + stats_emit_arena_hpa_slab_json(emitter, &sfull, "full_slabs"); + col_size.str_val = "full"; + stats_hpa_slab_cols_set(&sfull, &col_npageslabs_huge, &col_nactive_huge, + &col_ndirty_huge, &col_npageslabs_nonhuge, &col_nactive_nonhuge, + &col_ndirty_nonhuge, &col_nretained_nonhuge); + emitter_table_row(emitter, &row); + + stats_arena_hpa_slab_t sempty; + stats_gather_arena_hpa_slab(i, "empty_slabs", &sempty); + stats_emit_arena_hpa_slab_json(emitter, &sempty, "empty_slabs"); + col_size.str_val = "empty"; + stats_hpa_slab_cols_set(&sempty, &col_npageslabs_huge, &col_nactive_huge, + &col_ndirty_huge, &col_npageslabs_nonhuge, &col_nactive_nonhuge, + &col_ndirty_nonhuge, &col_nretained_nonhuge); + emitter_table_row(emitter, &row); + + /* Numeric size/ind for the per-size-class rows below. */ + col_size.type = emitter_type_size; + col_ind.type = emitter_type_unsigned; + size_t stats_arenas_mib[CTL_MAX_DEPTH]; CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas"); stats_arenas_mib[2] = i; CTL_LEAF_PREPARE(stats_arenas_mib, 3, "hpa_shard.nonfull_slabs"); - emitter_table_printf(emitter, " In nonfull slabs:\n"); - emitter_table_row(emitter, &header_row); emitter_json_array_kv_begin(emitter, "nonfull_slabs"); emitter_table_sparse_begin(emitter); for (pszind_t j = 0; j < PSSET_NPSIZES && j < SC_NPSIZES; j++) { From bf234f6b53b0da296e68c77aebbf32da8bb071fe Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Thu, 16 Jul 2026 06:33:32 -0700 Subject: [PATCH 5/9] Format change: label size-class ranges in the per-class stats tables Table-output change: label size classes above 8*PAGE as a range "prev_size+1..size" (e.g. "32769..40960") in the extents / lextents / hpa-nonfull tables, instead of a single size. Text only; JSON is unchanged (the ordered size-class metadata already implies the range). --- src/stats.c | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/stats.c b/src/stats.c index 1bf0a4da..ac322c11 100644 --- a/src/stats.c +++ b/src/stats.c @@ -702,6 +702,26 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, #undef EMITTER_TYPE_uint64_t } +/* + * Set a per-size-class row's "size" column. Above the slow-growth threshold, + * display the bucket bounds as "(prev_size,size]". This notation does not + * imply that every integer in the interval is a valid usable size. Smaller + * classes are displayed as a single size. buf must outlive row emission. + * (This is table labeling only; the ordered JSON arrays imply the bounds.) + */ +static void +stats_size_col_set(emitter_col_t *col, size_t size, size_t prev_size, char *buf, + size_t bufsz) { + if (size > USIZE_GROW_SLOW_THRESHOLD) { + malloc_snprintf(buf, bufsz, "(%zu,%zu]", prev_size, size); + col->type = emitter_type_title; + col->str_val = buf; + } else { + col->type = emitter_type_size; + col->size_val = size; + } +} + JEMALLOC_COLD static void stats_arena_bins_print( @@ -1012,6 +1032,15 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.lextents"); } + /* + * Carry the previous size class's size forward for the "(prev,size]" + * range label instead of looking it up by a large size-class index: + * with large size classes disabled (the default) that index may be out + * of range, and indexing sz_index2size_tab there would be unsafe. The + * class before the first large one is the last bin; sz_index2size_compute + * derives it arithmetically (no table access). + */ + size_t prev_size = sz_index2size_compute(nbins - 1); emitter_table_sparse_begin(emitter); for (j = 0; j < nlextents; j++) { stats_arena_lextent_t lext; @@ -1020,7 +1049,10 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { bool is_gap = (lext.nrequests == 0); - col_size.size_val = lext.lextent_size; + char size_buf[48]; + stats_size_col_set(&col_size, lext.lextent_size, prev_size, + size_buf, sizeof(size_buf)); + prev_size = lext.lextent_size; col_ind.unsigned_val = nbins + j; col_allocated.size_val = lext.curlextents * lext.lextent_size; col_nmalloc.uint64_val = lext.nmalloc; @@ -1106,7 +1138,9 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { &e.pinned_bytes); emitter_json_object_end(emitter); - col_size.size_val = sz_pind2sz(j); + char size_buf[48]; + stats_size_col_set(&col_size, sz_pind2sz(j), + j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); col_ind.size_val = j; col_ndirty.size_val = e.ndirty; col_dirty.size_val = e.dirty_bytes; @@ -1413,7 +1447,9 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { bool is_gap = (s.npageslabs_huge == 0 && s.npageslabs_nonhuge == 0); - col_size.size_val = sz_pind2sz(j); + char size_buf[48]; + stats_size_col_set(&col_size, sz_pind2sz(j), + j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); col_ind.size_val = j; col_npageslabs_huge.size_val = s.npageslabs_huge; col_nactive_huge.size_val = s.nactive_huge; From 1f3c2b066bc9c5fe978eface78c086232495b8a7 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Thu, 16 Jul 2026 08:59:42 -0700 Subject: [PATCH 6/9] Deduplicate mutex stats reader: merge identical global/arena helpers mutex_stats_read_global and mutex_stats_read_arena had byte-identical bodies; merge them into a single mutex_stats_read used by both the global and per-arena mutex tables. Pure cleanup; no functional or output change. --- src/stats.c | 43 ++++++++----------------------------------- 1 file changed, 8 insertions(+), 35 deletions(-) diff --git a/src/stats.c b/src/stats.c index ac322c11..4db836a0 100644 --- a/src/stats.c +++ b/src/stats.c @@ -579,40 +579,13 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name, col_uint64_t[mutex_counter_total_wait_time_ps].width = 10; } +/* + * Read one mutex's counters (leaf "name" under the prepared mib) into the + * emitter columns. Shared by the global and per-arena mutex tables, whose rows + * differ only in the mib and mutex name they pass. + */ static void -mutex_stats_read_global(size_t mib[], size_t miblen, const char *name, - emitter_col_t *col_name, - emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { - CTL_LEAF_PREPARE(mib, miblen, name); - size_t miblen_name = miblen + 1; - - col_name->str_val = name; - - emitter_col_t *dst; -#define EMITTER_TYPE_uint32_t emitter_type_uint32 -#define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ - dst = &col_##counter_type[mutex_counter_##counter]; \ - dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - CTL_LEAF(mib, miblen_name, #counter, \ - (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = \ - &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = (counter_type)rate_per_second( \ - base->counter_type##_val, uptime); \ - } - MUTEX_PROF_COUNTERS -#undef OP -#undef EMITTER_TYPE_uint32_t -#undef EMITTER_TYPE_uint64_t -} - -static void -mutex_stats_read_arena(size_t mib[], size_t miblen, const char *name, +mutex_stats_read(size_t mib[], size_t miblen, const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], @@ -1518,7 +1491,7 @@ stats_arena_mutexes_print( i++) { const char *name = arena_mutex_names[i]; emitter_json_object_kv_begin(emitter, name); - mutex_stats_read_arena( + mutex_stats_read( stats_arenas_mib, 4, name, &col_name, col64, col32, uptime); mutex_stats_emit(emitter, &row, col64, col32); emitter_json_object_end(emitter); /* Close the mutex dict. */ @@ -2311,7 +2284,7 @@ stats_global_mutexes_print(emitter_t *emitter) { size_t stats_mutexes_mib[CTL_MAX_DEPTH]; CTL_LEAF_PREPARE(stats_mutexes_mib, 0, "stats.mutexes"); for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { - mutex_stats_read_global(stats_mutexes_mib, 2, + mutex_stats_read(stats_mutexes_mib, 2, global_mutex_names[i], &name, col64, col32, uptime); emitter_json_object_kv_begin(emitter, global_mutex_names[i]); mutex_stats_emit(emitter, &row, col64, col32); From 02f8e972fe7caf6df47851320c283bc76537bd22 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Thu, 16 Jul 2026 08:59:54 -0700 Subject: [PATCH 7/9] Fix col_ind to write unsigned_val, matching its column type The extents and hpa-nonfull "ind" columns are emitter_type_unsigned but were set via col_ind.size_val -- correct only by little-endian union overlap. Output is byte-identical. Pre-existing (predates the refactor). --- src/stats.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stats.c b/src/stats.c index 4db836a0..3a8b5263 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1114,7 +1114,7 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { char size_buf[48]; stats_size_col_set(&col_size, sz_pind2sz(j), j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); - col_ind.size_val = j; + col_ind.unsigned_val = j; col_ndirty.size_val = e.ndirty; col_dirty.size_val = e.dirty_bytes; col_nmuzzy.size_val = e.nmuzzy; @@ -1423,7 +1423,7 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { char size_buf[48]; stats_size_col_set(&col_size, sz_pind2sz(j), j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); - col_ind.size_val = j; + col_ind.unsigned_val = j; col_npageslabs_huge.size_val = s.npageslabs_huge; col_nactive_huge.size_val = s.nactive_huge; col_ndirty_huge.size_val = s.ndirty_huge; From e1b5edcc62c4ddfb53679ede851aad8731ed00b8 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 17 Jul 2026 09:02:53 -0700 Subject: [PATCH 8/9] Refactor stats tables around descriptor-driven row emission --- include/jemalloc/internal/emitter.h | 90 +++ src/stats.c | 961 +++++++++++++++++----------- test/unit/emitter.c | 109 +++- 3 files changed, 802 insertions(+), 358 deletions(-) diff --git a/include/jemalloc/internal/emitter.h b/include/jemalloc/internal/emitter.h index 9e645106..359077e6 100644 --- a/include/jemalloc/internal/emitter.h +++ b/include/jemalloc/internal/emitter.h @@ -590,6 +590,96 @@ emitter_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) { } } +/* + * Descriptor-driven column tables. A descriptor array defines table order; + * callers provide a separate index array when JSON requires a different + * subset or ordering. The getter lets one engine serve different row types + * while keeping derived values (rates, utilization, and so on) with the + * table-specific code. A column is active when all its required_flags are + * present in the active_flags supplied by the caller; the emitter assigns no + * domain-specific meaning to those bits. + */ +typedef struct emitter_col_desc_s emitter_col_desc_t; +struct emitter_col_desc_s { + const char *json_key; + const char *table_label; + emitter_justify_t justify; + int width; + emitter_type_t type; + unsigned required_flags; + void (*get)(const void *row, emitter_col_t *col); +}; + +static inline bool +emitter_col_desc_active(const emitter_col_desc_t *desc, + unsigned active_flags) { + return (desc->required_flags & ~active_flags) == 0; +} + +static inline void +emitter_col_table_build(const emitter_col_desc_t *descs, size_t ndescs, + unsigned active_flags, emitter_row_t *row, emitter_col_t *cols, + emitter_row_t *header_row, emitter_col_t *header_cols) { + emitter_row_init(row); + emitter_row_init(header_row); + for (size_t i = 0; i < ndescs; i++) { + if (!emitter_col_desc_active(&descs[i], active_flags)) { + continue; + } + + emitter_col_init(&cols[i], row); + cols[i].justify = descs[i].justify; + cols[i].width = descs[i].width; + cols[i].type = descs[i].type; + + emitter_col_init(&header_cols[i], header_row); + header_cols[i].justify = descs[i].justify; + header_cols[i].width = descs[i].width; + header_cols[i].type = emitter_type_title; + header_cols[i].str_val = descs[i].table_label != NULL + ? descs[i].table_label : descs[i].json_key; + } +} + +static inline void +emitter_col_table_header(emitter_t *emitter, emitter_row_t *header_row, + emitter_col_t *first_header_col, const char *table_prefix, + const char *json_key) { + first_header_col->width -= (int)strlen(table_prefix); + emitter_table_printf(emitter, "%s", table_prefix); + emitter_table_row(emitter, header_row); + emitter_json_array_kv_begin(emitter, json_key); +} + +static inline void +emitter_col_table_fill(const emitter_col_desc_t *descs, size_t ndescs, + unsigned active_flags, emitter_col_t *cols, const void *row) { + for (size_t i = 0; i < ndescs; i++) { + if (!emitter_col_desc_active(&descs[i], active_flags)) { + continue; + } + cols[i].type = descs[i].type; + descs[i].get(row, &cols[i]); + } +} + +static inline void +emitter_col_table_emit_json(emitter_t *emitter, + const emitter_col_desc_t *descs, size_t ndescs, unsigned active_flags, + emitter_col_t *cols, const unsigned *json_order, size_t json_order_len) { + for (size_t i = 0; i < json_order_len; i++) { + unsigned col_ind = json_order[i]; + assert(col_ind < ndescs); + const emitter_col_desc_t *desc = &descs[col_ind]; + if (!emitter_col_desc_active(desc, active_flags)) { + continue; + } + assert(desc->json_key != NULL); + emitter_json_kv(emitter, desc->json_key, cols[col_ind].type, + (const void *)&cols[col_ind].bool_val); + } +} + static inline void emitter_begin(emitter_t *emitter) { if (emitter_outputs_json(emitter)) { diff --git a/src/stats.c b/src/stats.c index 3a8b5263..8ae8ca94 100644 --- a/src/stats.c +++ b/src/stats.c @@ -695,18 +695,223 @@ stats_size_col_set(emitter_col_t *col, size_t size, size_t prev_size, char *buf, } } +enum { + STATS_COL_FLAG_NONE = 0, + STATS_COL_FLAG_PROF = 1U << 0, +}; + +static unsigned +stats_col_active_flags(bool prof_stats_on) { + return prof_stats_on ? STATS_COL_FLAG_PROF : STATS_COL_FLAG_NONE; +} + +typedef struct { + const stats_arena_bin_t *bin; + unsigned ind; + size_t page; + uint64_t uptime; + const char *util; +} stats_arena_bin_emit_row_t; + +#define BIN_COL_GET(name, value_member, field) \ + static void \ + stats_bin_col_get_##name(const void *vrow, emitter_col_t *col) { \ + const stats_arena_bin_emit_row_t *row = vrow; \ + col->value_member = row->bin->field; \ + } +#define BIN_COL_GET_RATE(name, field) \ + static void \ + stats_bin_col_get_##name(const void *vrow, emitter_col_t *col) { \ + const stats_arena_bin_emit_row_t *row = vrow; \ + col->uint64_val = rate_per_second( \ + row->bin->field, row->uptime); \ + } +BIN_COL_GET(size, size_val, reg_size) +BIN_COL_GET(nmalloc, uint64_val, nmalloc) +BIN_COL_GET(ndalloc, uint64_val, ndalloc) +BIN_COL_GET(nrequests, uint64_val, nrequests) +BIN_COL_GET(prof_live_requested, uint64_val, prof_live.req_sum) +BIN_COL_GET(prof_live_count, uint64_val, prof_live.count) +BIN_COL_GET(prof_accum_requested, uint64_val, prof_accum.req_sum) +BIN_COL_GET(prof_accum_count, uint64_val, prof_accum.count) +BIN_COL_GET(nshards, unsigned_val, nshards) +BIN_COL_GET(curregs, size_val, curregs) +BIN_COL_GET(curslabs, size_val, curslabs) +BIN_COL_GET(nonfull_slabs, size_val, nonfull_slabs) +BIN_COL_GET(regs, unsigned_val, nregs) +BIN_COL_GET(nfills, uint64_val, nfills) +BIN_COL_GET(nflushes, uint64_val, nflushes) +BIN_COL_GET(nslabs, uint64_val, nslabs) +BIN_COL_GET(nreslabs, uint64_val, nreslabs) +BIN_COL_GET_RATE(nmalloc_ps, nmalloc) +BIN_COL_GET_RATE(ndalloc_ps, ndalloc) +BIN_COL_GET_RATE(nrequests_ps, nrequests) +BIN_COL_GET_RATE(nfills_ps, nfills) +BIN_COL_GET_RATE(nflushes_ps, nflushes) +BIN_COL_GET_RATE(nreslabs_ps, nreslabs) +#undef BIN_COL_GET +#undef BIN_COL_GET_RATE + +static void +stats_bin_col_get_ind(const void *vrow, emitter_col_t *col) { + const stats_arena_bin_emit_row_t *row = vrow; + col->unsigned_val = row->ind; +} + +static void +stats_bin_col_get_allocated(const void *vrow, emitter_col_t *col) { + const stats_arena_bin_emit_row_t *row = vrow; + col->size_val = row->bin->curregs * row->bin->reg_size; +} + +static void +stats_bin_col_get_pgs(const void *vrow, emitter_col_t *col) { + const stats_arena_bin_emit_row_t *row = vrow; + col->size_val = row->bin->slab_size / row->page; +} + +static void +stats_bin_col_get_spacer(const void *vrow, emitter_col_t *col) { + (void)vrow; + col->str_val = " "; +} + +static void +stats_bin_col_get_util(const void *vrow, emitter_col_t *col) { + const stats_arena_bin_emit_row_t *row = vrow; + col->str_val = row->util; +} + +enum { + BIN_COL_SIZE, + BIN_COL_IND, + BIN_COL_ALLOCATED, + BIN_COL_NMALLOC, + BIN_COL_NMALLOC_PS, + BIN_COL_NDALLOC, + BIN_COL_NDALLOC_PS, + BIN_COL_NREQUESTS, + BIN_COL_NREQUESTS_PS, + BIN_COL_PROF_LIVE_REQUESTED, + BIN_COL_PROF_LIVE_COUNT, + BIN_COL_PROF_ACCUM_REQUESTED, + BIN_COL_PROF_ACCUM_COUNT, + BIN_COL_NSHARDS, + BIN_COL_CURREGS, + BIN_COL_CURSLABS, + BIN_COL_NONFULL_SLABS, + BIN_COL_REGS, + BIN_COL_PGS, + BIN_COL_SPACER, + BIN_COL_UTIL, + BIN_COL_NFILLS, + BIN_COL_NFILLS_PS, + BIN_COL_NFLUSHES, + BIN_COL_NFLUSHES_PS, + BIN_COL_NSLABS, + BIN_COL_NRESLABS, + BIN_COL_NRESLABS_PS, + BIN_COL_COUNT +}; + +#define BIN_DESC(key, label, width, type, flags, name) \ + {key, label, emitter_justify_right, width, emitter_type_##type, flags, \ + stats_bin_col_get_##name} +static const emitter_col_desc_t stats_bin_cols[] = { + BIN_DESC(NULL, "size", 20, size, STATS_COL_FLAG_NONE, size), + BIN_DESC(NULL, "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind), + BIN_DESC(NULL, "allocated", 14, size, STATS_COL_FLAG_NONE, allocated), + BIN_DESC("nmalloc", "nmalloc", 14, uint64, STATS_COL_FLAG_NONE, + nmalloc), + BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, nmalloc_ps), + BIN_DESC("ndalloc", "ndalloc", 14, uint64, STATS_COL_FLAG_NONE, + ndalloc), + BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, ndalloc_ps), + BIN_DESC("nrequests", "nrequests", 15, uint64, STATS_COL_FLAG_NONE, + nrequests), + BIN_DESC(NULL, "(#/sec)", 10, uint64, STATS_COL_FLAG_NONE, + nrequests_ps), + BIN_DESC("prof_live_requested", "prof_live_requested", 21, uint64, + STATS_COL_FLAG_PROF, prof_live_requested), + BIN_DESC("prof_live_count", "prof_live_count", 17, uint64, + STATS_COL_FLAG_PROF, prof_live_count), + BIN_DESC("prof_accum_requested", "prof_accum_requested", 21, uint64, + STATS_COL_FLAG_PROF, prof_accum_requested), + BIN_DESC("prof_accum_count", "prof_accum_count", 17, uint64, + STATS_COL_FLAG_PROF, prof_accum_count), + BIN_DESC(NULL, "nshards", 9, unsigned, STATS_COL_FLAG_NONE, nshards), + BIN_DESC("curregs", "curregs", 13, size, STATS_COL_FLAG_NONE, curregs), + BIN_DESC("curslabs", "curslabs", 13, size, STATS_COL_FLAG_NONE, + curslabs), + BIN_DESC("nonfull_slabs", "nonfull_slabs", 15, size, + STATS_COL_FLAG_NONE, + nonfull_slabs), + BIN_DESC(NULL, "regs", 5, unsigned, STATS_COL_FLAG_NONE, regs), + BIN_DESC(NULL, "pgs", 4, size, STATS_COL_FLAG_NONE, pgs), + BIN_DESC(NULL, " ", 1, title, STATS_COL_FLAG_NONE, spacer), + BIN_DESC(NULL, "util", 6, title, STATS_COL_FLAG_NONE, util), + BIN_DESC("nfills", "nfills", 13, uint64, STATS_COL_FLAG_NONE, nfills), + BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, nfills_ps), + BIN_DESC("nflushes", "nflushes", 13, uint64, STATS_COL_FLAG_NONE, + nflushes), + BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + nflushes_ps), + BIN_DESC(NULL, "nslabs", 13, uint64, STATS_COL_FLAG_NONE, nslabs), + BIN_DESC("nreslabs", "nreslabs", 13, uint64, STATS_COL_FLAG_NONE, + nreslabs), + BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + nreslabs_ps), +}; +#undef BIN_DESC +_Static_assert(sizeof(stats_bin_cols) / sizeof(stats_bin_cols[0]) == + BIN_COL_COUNT, "stats_bin_cols must match BIN_COL_COUNT"); + +static const unsigned stats_bin_json_order[] = { + BIN_COL_NMALLOC, + BIN_COL_NDALLOC, + BIN_COL_CURREGS, + BIN_COL_NREQUESTS, + BIN_COL_PROF_LIVE_REQUESTED, + BIN_COL_PROF_LIVE_COUNT, + BIN_COL_PROF_ACCUM_REQUESTED, + BIN_COL_PROF_ACCUM_COUNT, + BIN_COL_NFILLS, + BIN_COL_NFLUSHES, + BIN_COL_NRESLABS, + BIN_COL_CURSLABS, + BIN_COL_NONFULL_SLABS, +}; + +static void +stats_emit_arena_bin_row(emitter_t *emitter, emitter_row_t *table_row, + emitter_col_t *cols, const stats_arena_bin_emit_row_t *row, + bool prof_stats_on, bool mutex, emitter_col_t *mutex64, + emitter_col_t *mutex32, bool is_gap) { + unsigned active_flags = stats_col_active_flags(prof_stats_on); + emitter_col_table_fill( + stats_bin_cols, BIN_COL_COUNT, active_flags, cols, row); + emitter_json_object_begin(emitter); + emitter_col_table_emit_json(emitter, stats_bin_cols, BIN_COL_COUNT, + active_flags, cols, stats_bin_json_order, + sizeof(stats_bin_json_order) / sizeof(stats_bin_json_order[0])); + if (mutex) { + emitter_json_object_kv_begin(emitter, "mutex"); + mutex_stats_emit(emitter, NULL, mutex64, mutex32); + emitter_json_object_end(emitter); + } + emitter_json_object_end(emitter); + emitter_table_sparse_row(emitter, table_row, is_gap); +} + JEMALLOC_COLD static void stats_arena_bins_print( emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { /* - * Streaming table (deviates from the gather/emit split): the SC_NBINS - * rows are gathered and emitted one at a time (stats_gather_arena_bin - * per row + inline emit, with all-zero runs collapsed to "---"). This - * preserves O(1) memory. Full gather-all-rows-before-emit separation - * would require buffering the table; a separate per-row emit helper - * would not. The lextents / extents / hpa-nonfull tables follow the - * same pattern. + * Process one size-class row at a time to preserve O(1) memory. Full + * gather-all-rows-before-emit separation would require buffering the + * table, so gathering and emission remain interleaved across rows. The + * lextents, extents, and HPA nonfull tables follow the same pattern. */ size_t page; unsigned nbins, j; @@ -715,54 +920,13 @@ stats_arena_bins_print( CTL_GET("arenas.nbins", &nbins, unsigned); - emitter_row_t header_row; - emitter_row_init(&header_row); - - emitter_row_t row; - emitter_row_init(&row); - bool prof_stats_on = config_prof && opt_prof && opt_prof_stats && i == MALLCTL_ARENAS_ALL; - - COL_HDR(row, size, NULL, right, 20, size) - COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, allocated, NULL, right, 14, size) - COL_HDR(row, nmalloc, NULL, right, 14, uint64) - COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, ndalloc, NULL, right, 14, uint64) - COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nrequests, NULL, right, 15, uint64) - COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64) - COL_HDR_DECLARE(prof_live_requested); - COL_HDR_DECLARE(prof_live_count); - COL_HDR_DECLARE(prof_accum_requested); - COL_HDR_DECLARE(prof_accum_count); - if (prof_stats_on) { - COL_HDR_INIT(row, prof_live_requested, NULL, right, 21, uint64) - COL_HDR_INIT(row, prof_live_count, NULL, right, 17, uint64) - COL_HDR_INIT(row, prof_accum_requested, NULL, right, 21, uint64) - COL_HDR_INIT(row, prof_accum_count, NULL, right, 17, uint64) - } - COL_HDR(row, nshards, NULL, right, 9, unsigned) - COL_HDR(row, curregs, NULL, right, 13, size) - COL_HDR(row, curslabs, NULL, right, 13, size) - COL_HDR(row, nonfull_slabs, NULL, right, 15, size) - COL_HDR(row, regs, NULL, right, 5, unsigned) - COL_HDR(row, pgs, NULL, right, 4, size) - /* To buffer a right- and left-justified column. */ - COL_HDR(row, justify_spacer, NULL, right, 1, title) - COL_HDR(row, util, NULL, right, 6, title) - COL_HDR(row, nfills, NULL, right, 13, uint64) - COL_HDR(row, nfills_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nflushes, NULL, right, 13, uint64) - COL_HDR(row, nflushes_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nslabs, NULL, right, 13, uint64) - COL_HDR(row, nreslabs, NULL, right, 13, uint64) - COL_HDR(row, nreslabs_ps, "(#/sec)", right, 8, uint64) - - /* Don't want to actually print the name. */ - header_justify_spacer.str_val = " "; - col_justify_spacer.str_val = " "; + emitter_row_t row, header_row; + emitter_col_t cols[BIN_COL_COUNT], header_cols[BIN_COL_COUNT]; + emitter_col_table_build(stats_bin_cols, BIN_COL_COUNT, + stats_col_active_flags(prof_stats_on), &row, cols, &header_row, + header_cols); emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters]; emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters]; @@ -777,14 +941,8 @@ stats_arena_bins_print( &header_row, NULL, NULL, header_mutex64, header_mutex32); } - /* - * We print a "bins:" header as part of the table row; we need to adjust - * the header size column to compensate. - */ - header_size.width -= 5; - emitter_table_printf(emitter, "bins:"); - emitter_table_row(emitter, &header_row); - emitter_json_array_kv_begin(emitter, "bins"); + emitter_col_table_header(emitter, &header_row, + &header_cols[BIN_COL_SIZE], "bins:", "bins"); size_t stats_arenas_mib[CTL_MAX_DEPTH]; CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas"); @@ -827,15 +985,7 @@ stats_arena_bins_print( continue; } - /* - * Per-size-class tables stream: gather one row, emit it, reuse - * buffers. Full gather-all-rows-before-emit separation would - * require table-sized storage, so gathering and emission remain - * interleaved across rows. In particular the per-bin mutex - * counters are read here (the mutex subsystem reads straight into - * emitter columns) rather than up front. A separate per-row emit - * helper could further decouple the code without buffering. - */ + /* Mutex counters still gather directly into their emitter columns. */ stats_gather_arena_bin(stats_arenas_mib, arenas_bin_mib, &bin); if (mutex) { @@ -843,43 +993,6 @@ stats_arena_bins_print( col_mutex64, col_mutex32, uptime); } - emitter_json_object_begin(emitter); - emitter_json_kv( - emitter, "nmalloc", emitter_type_uint64, &bin.nmalloc); - emitter_json_kv( - emitter, "ndalloc", emitter_type_uint64, &bin.ndalloc); - emitter_json_kv( - emitter, "curregs", emitter_type_size, &bin.curregs); - emitter_json_kv( - emitter, "nrequests", emitter_type_uint64, &bin.nrequests); - if (prof_stats_on) { - emitter_json_kv(emitter, "prof_live_requested", - emitter_type_uint64, &bin.prof_live.req_sum); - emitter_json_kv(emitter, "prof_live_count", - emitter_type_uint64, &bin.prof_live.count); - emitter_json_kv(emitter, "prof_accum_requested", - emitter_type_uint64, &bin.prof_accum.req_sum); - emitter_json_kv(emitter, "prof_accum_count", - emitter_type_uint64, &bin.prof_accum.count); - } - emitter_json_kv( - emitter, "nfills", emitter_type_uint64, &bin.nfills); - emitter_json_kv( - emitter, "nflushes", emitter_type_uint64, &bin.nflushes); - emitter_json_kv( - emitter, "nreslabs", emitter_type_uint64, &bin.nreslabs); - emitter_json_kv( - emitter, "curslabs", emitter_type_size, &bin.curslabs); - emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size, - &bin.nonfull_slabs); - if (mutex) { - emitter_json_object_kv_begin(emitter, "mutex"); - mutex_stats_emit( - emitter, NULL, col_mutex64, col_mutex32); - emitter_json_object_end(emitter); - } - emitter_json_object_end(emitter); - size_t availregs = bin.nregs * bin.curslabs; char util[6]; if (get_rate_str( @@ -899,50 +1012,140 @@ stats_arena_bins_print( } } - col_size.size_val = bin.reg_size; - col_ind.unsigned_val = j; - col_allocated.size_val = bin.curregs * bin.reg_size; - col_nmalloc.uint64_val = bin.nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(bin.nmalloc, uptime); - col_ndalloc.uint64_val = bin.ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(bin.ndalloc, uptime); - col_nrequests.uint64_val = bin.nrequests; - col_nrequests_ps.uint64_val = rate_per_second( - bin.nrequests, uptime); - if (prof_stats_on) { - col_prof_live_requested.uint64_val = bin.prof_live.req_sum; - col_prof_live_count.uint64_val = bin.prof_live.count; - col_prof_accum_requested.uint64_val = - bin.prof_accum.req_sum; - col_prof_accum_count.uint64_val = bin.prof_accum.count; - } - col_nshards.unsigned_val = bin.nshards; - col_curregs.size_val = bin.curregs; - col_curslabs.size_val = bin.curslabs; - col_nonfull_slabs.size_val = bin.nonfull_slabs; - col_regs.unsigned_val = bin.nregs; - col_pgs.size_val = bin.slab_size / page; - col_util.str_val = util; - col_nfills.uint64_val = bin.nfills; - col_nfills_ps.uint64_val = rate_per_second(bin.nfills, uptime); - col_nflushes.uint64_val = bin.nflushes; - col_nflushes_ps.uint64_val = rate_per_second(bin.nflushes, uptime); - col_nslabs.uint64_val = bin.nslabs; - col_nreslabs.uint64_val = bin.nreslabs; - col_nreslabs_ps.uint64_val = rate_per_second(bin.nreslabs, uptime); - - /* - * Note that mutex columns were initialized above, if mutex == - * true. - */ - - emitter_table_sparse_row(emitter, &row, is_gap); + stats_arena_bin_emit_row_t emit_row = { + &bin, j, page, uptime, util}; + stats_emit_arena_bin_row(emitter, &row, cols, &emit_row, + prof_stats_on, mutex, col_mutex64, col_mutex32, is_gap); } emitter_json_array_end(emitter); /* Close "bins". */ emitter_table_sparse_end(emitter); } +typedef struct { + const stats_arena_lextent_t *lextent; + unsigned ind; + uint64_t uptime; +} stats_arena_lextent_emit_row_t; + +#define LEXTENT_COL_GET(name, value_member, field) \ + static void \ + stats_lextent_col_get_##name(const void *vrow, emitter_col_t *col) { \ + const stats_arena_lextent_emit_row_t *row = vrow; \ + col->value_member = row->lextent->field; \ + } +#define LEXTENT_COL_GET_RATE(name, field) \ + static void \ + stats_lextent_col_get_##name(const void *vrow, emitter_col_t *col) { \ + const stats_arena_lextent_emit_row_t *row = vrow; \ + col->uint64_val = rate_per_second( \ + row->lextent->field, row->uptime); \ + } +LEXTENT_COL_GET(size, size_val, lextent_size) +LEXTENT_COL_GET(nmalloc, uint64_val, nmalloc) +LEXTENT_COL_GET(ndalloc, uint64_val, ndalloc) +LEXTENT_COL_GET(nrequests, uint64_val, nrequests) +LEXTENT_COL_GET(prof_live_requested, uint64_val, prof_live.req_sum) +LEXTENT_COL_GET(prof_live_count, uint64_val, prof_live.count) +LEXTENT_COL_GET(prof_accum_requested, uint64_val, prof_accum.req_sum) +LEXTENT_COL_GET(prof_accum_count, uint64_val, prof_accum.count) +LEXTENT_COL_GET(curlextents, size_val, curlextents) +LEXTENT_COL_GET_RATE(nmalloc_ps, nmalloc) +LEXTENT_COL_GET_RATE(ndalloc_ps, ndalloc) +LEXTENT_COL_GET_RATE(nrequests_ps, nrequests) +#undef LEXTENT_COL_GET +#undef LEXTENT_COL_GET_RATE + +static void +stats_lextent_col_get_ind(const void *vrow, emitter_col_t *col) { + const stats_arena_lextent_emit_row_t *row = vrow; + col->unsigned_val = row->ind; +} + +static void +stats_lextent_col_get_allocated(const void *vrow, emitter_col_t *col) { + const stats_arena_lextent_emit_row_t *row = vrow; + col->size_val = row->lextent->curlextents * row->lextent->lextent_size; +} + +enum { + LEXTENT_COL_SIZE, + LEXTENT_COL_IND, + LEXTENT_COL_ALLOCATED, + LEXTENT_COL_NMALLOC, + LEXTENT_COL_NMALLOC_PS, + LEXTENT_COL_NDALLOC, + LEXTENT_COL_NDALLOC_PS, + LEXTENT_COL_NREQUESTS, + LEXTENT_COL_NREQUESTS_PS, + LEXTENT_COL_PROF_LIVE_REQUESTED, + LEXTENT_COL_PROF_LIVE_COUNT, + LEXTENT_COL_PROF_ACCUM_REQUESTED, + LEXTENT_COL_PROF_ACCUM_COUNT, + LEXTENT_COL_CURLEXTENTS, + LEXTENT_COL_COUNT +}; + +#define LEXTENT_DESC(key, label, width, type, flags, name) \ + {key, label, emitter_justify_right, width, emitter_type_##type, flags, \ + stats_lextent_col_get_##name} +static const emitter_col_desc_t stats_lextent_cols[] = { + LEXTENT_DESC(NULL, "size", 20, size, STATS_COL_FLAG_NONE, size), + LEXTENT_DESC(NULL, "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind), + LEXTENT_DESC(NULL, "allocated", 13, size, STATS_COL_FLAG_NONE, + allocated), + LEXTENT_DESC(NULL, "nmalloc", 13, uint64, STATS_COL_FLAG_NONE, nmalloc), + LEXTENT_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + nmalloc_ps), + LEXTENT_DESC(NULL, "ndalloc", 13, uint64, STATS_COL_FLAG_NONE, ndalloc), + LEXTENT_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + ndalloc_ps), + LEXTENT_DESC(NULL, "nrequests", 13, uint64, STATS_COL_FLAG_NONE, + nrequests), + LEXTENT_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + nrequests_ps), + LEXTENT_DESC("prof_live_requested", "prof_live_requested", 21, uint64, + STATS_COL_FLAG_PROF, prof_live_requested), + LEXTENT_DESC("prof_live_count", "prof_live_count", 17, uint64, + STATS_COL_FLAG_PROF, prof_live_count), + LEXTENT_DESC("prof_accum_requested", "prof_accum_requested", 21, + uint64, STATS_COL_FLAG_PROF, prof_accum_requested), + LEXTENT_DESC("prof_accum_count", "prof_accum_count", 17, uint64, + STATS_COL_FLAG_PROF, prof_accum_count), + LEXTENT_DESC("curlextents", "curlextents", 13, size, + STATS_COL_FLAG_NONE, curlextents), +}; +#undef LEXTENT_DESC +_Static_assert(sizeof(stats_lextent_cols) / sizeof(stats_lextent_cols[0]) == + LEXTENT_COL_COUNT, "stats_lextent_cols must match LEXTENT_COL_COUNT"); + +static const unsigned stats_lextent_json_order[] = { + LEXTENT_COL_PROF_LIVE_REQUESTED, + LEXTENT_COL_PROF_LIVE_COUNT, + LEXTENT_COL_PROF_ACCUM_REQUESTED, + LEXTENT_COL_PROF_ACCUM_COUNT, + LEXTENT_COL_CURLEXTENTS, +}; + +static void +stats_emit_arena_lextent_row(emitter_t *emitter, emitter_row_t *table_row, + emitter_col_t *cols, const stats_arena_lextent_emit_row_t *row, + bool prof_stats_on, size_t prev_size, bool is_gap) { + unsigned active_flags = stats_col_active_flags(prof_stats_on); + emitter_col_table_fill(stats_lextent_cols, LEXTENT_COL_COUNT, + active_flags, cols, row); + char size_buf[48]; + stats_size_col_set(&cols[LEXTENT_COL_SIZE], row->lextent->lextent_size, + prev_size, size_buf, sizeof(size_buf)); + emitter_json_object_begin(emitter); + emitter_col_table_emit_json(emitter, stats_lextent_cols, + LEXTENT_COL_COUNT, active_flags, cols, stats_lextent_json_order, + sizeof(stats_lextent_json_order) / + sizeof(stats_lextent_json_order[0])); + emitter_table_sparse_row(emitter, table_row, is_gap); + emitter_json_object_end(emitter); +} + JEMALLOC_COLD static void stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { @@ -952,45 +1155,15 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { CTL_GET("arenas.nbins", &nbins, unsigned); CTL_GET("arenas.nlextents", &nlextents, unsigned); - emitter_row_t header_row; - emitter_row_init(&header_row); - emitter_row_t row; - emitter_row_init(&row); - bool prof_stats_on = config_prof && opt_prof && opt_prof_stats && i == MALLCTL_ARENAS_ALL; - - COL_HDR(row, size, NULL, right, 20, size) - COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, allocated, NULL, right, 13, size) - COL_HDR(row, nmalloc, NULL, right, 13, uint64) - COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, ndalloc, NULL, right, 13, uint64) - COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) - COL_HDR(row, nrequests, NULL, right, 13, uint64) - COL_HDR(row, nrequests_ps, "(#/sec)", right, 8, uint64) - COL_HDR_DECLARE(prof_live_requested) - COL_HDR_DECLARE(prof_live_count) - COL_HDR_DECLARE(prof_accum_requested) - COL_HDR_DECLARE(prof_accum_count) - if (prof_stats_on) { - COL_HDR_INIT(row, prof_live_requested, NULL, right, 21, uint64) - COL_HDR_INIT(row, prof_live_count, NULL, right, 17, uint64) - COL_HDR_INIT(row, prof_accum_requested, NULL, right, 21, uint64) - COL_HDR_INIT(row, prof_accum_count, NULL, right, 17, uint64) - col_prof_live_requested.json_key = "prof_live_requested"; - col_prof_live_count.json_key = "prof_live_count"; - col_prof_accum_requested.json_key = "prof_accum_requested"; - col_prof_accum_count.json_key = "prof_accum_count"; - } - COL_HDR(row, curlextents, NULL, right, 13, size) - col_curlextents.json_key = "curlextents"; - - /* As with bins, we label the large extents table. */ - header_size.width -= 6; - emitter_table_printf(emitter, "large:"); - emitter_table_row(emitter, &header_row); - emitter_json_array_kv_begin(emitter, "lextents"); + emitter_row_t row, header_row; + emitter_col_t cols[LEXTENT_COL_COUNT], header_cols[LEXTENT_COL_COUNT]; + emitter_col_table_build(stats_lextent_cols, LEXTENT_COL_COUNT, + stats_col_active_flags(prof_stats_on), &row, cols, &header_row, + header_cols); + emitter_col_table_header(emitter, &header_row, + &header_cols[LEXTENT_COL_SIZE], "large:", "lextents"); size_t stats_arenas_mib[CTL_MAX_DEPTH]; CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas"); @@ -1022,64 +1195,131 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { bool is_gap = (lext.nrequests == 0); - char size_buf[48]; - stats_size_col_set(&col_size, lext.lextent_size, prev_size, - size_buf, sizeof(size_buf)); + stats_arena_lextent_emit_row_t emit_row = { + &lext, nbins + j, uptime}; + stats_emit_arena_lextent_row(emitter, &row, cols, &emit_row, + prof_stats_on, prev_size, is_gap); prev_size = lext.lextent_size; - col_ind.unsigned_val = nbins + j; - col_allocated.size_val = lext.curlextents * lext.lextent_size; - col_nmalloc.uint64_val = lext.nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(lext.nmalloc, uptime); - col_ndalloc.uint64_val = lext.ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(lext.ndalloc, uptime); - col_nrequests.uint64_val = lext.nrequests; - col_nrequests_ps.uint64_val = rate_per_second( - lext.nrequests, uptime); - if (prof_stats_on) { - col_prof_live_requested.uint64_val = lext.prof_live.req_sum; - col_prof_live_count.uint64_val = lext.prof_live.count; - col_prof_accum_requested.uint64_val = - lext.prof_accum.req_sum; - col_prof_accum_count.uint64_val = lext.prof_accum.count; - } - col_curlextents.size_val = lext.curlextents; - - emitter_json_object_begin(emitter); - emitter_sparse_row(emitter, &row, is_gap); - emitter_json_object_end(emitter); } emitter_json_array_end(emitter); /* Close "lextents". */ emitter_table_sparse_end(emitter); } +typedef struct { + const stats_arena_extent_t *extent; + unsigned ind; + size_t size; +} stats_arena_extent_emit_row_t; + +#define EXTENT_COL_GET(name, field) \ + static void \ + stats_extent_col_get_##name(const void *vrow, emitter_col_t *col) { \ + const stats_arena_extent_emit_row_t *row = vrow; \ + col->size_val = row->extent->field; \ + } +EXTENT_COL_GET(ndirty, ndirty) +EXTENT_COL_GET(dirty, dirty_bytes) +EXTENT_COL_GET(nmuzzy, nmuzzy) +EXTENT_COL_GET(muzzy, muzzy_bytes) +EXTENT_COL_GET(nretained, nretained) +EXTENT_COL_GET(retained, retained_bytes) +EXTENT_COL_GET(npinned, npinned) +EXTENT_COL_GET(pinned, pinned_bytes) +EXTENT_COL_GET(ntotal, ntotal) +EXTENT_COL_GET(total, total_bytes) +#undef EXTENT_COL_GET + +static void +stats_extent_col_get_size(const void *vrow, emitter_col_t *col) { + const stats_arena_extent_emit_row_t *row = vrow; + col->size_val = row->size; +} + +static void +stats_extent_col_get_ind(const void *vrow, emitter_col_t *col) { + const stats_arena_extent_emit_row_t *row = vrow; + col->unsigned_val = row->ind; +} + +enum { + EXTENT_COL_SIZE, + EXTENT_COL_IND, + EXTENT_COL_NDIRTY, + EXTENT_COL_DIRTY, + EXTENT_COL_NMUZZY, + EXTENT_COL_MUZZY, + EXTENT_COL_NRETAINED, + EXTENT_COL_RETAINED, + EXTENT_COL_NPINNED, + EXTENT_COL_PINNED, + EXTENT_COL_NTOTAL, + EXTENT_COL_TOTAL, + EXTENT_COL_COUNT +}; + +#define EXTENT_DESC(key, label, name) \ + {key, label, emitter_justify_right, 13, emitter_type_size, \ + STATS_COL_FLAG_NONE, stats_extent_col_get_##name} +static const emitter_col_desc_t stats_extent_cols[] = { + {NULL, "size", emitter_justify_right, 20, emitter_type_size, + STATS_COL_FLAG_NONE, stats_extent_col_get_size}, + {NULL, "ind", emitter_justify_right, 4, emitter_type_unsigned, + STATS_COL_FLAG_NONE, stats_extent_col_get_ind}, + EXTENT_DESC("ndirty", "ndirty", ndirty), + EXTENT_DESC("dirty_bytes", "dirty", dirty), + EXTENT_DESC("nmuzzy", "nmuzzy", nmuzzy), + EXTENT_DESC("muzzy_bytes", "muzzy", muzzy), + EXTENT_DESC("nretained", "nretained", nretained), + EXTENT_DESC("retained_bytes", "retained", retained), + EXTENT_DESC("npinned", "npinned", npinned), + EXTENT_DESC("pinned_bytes", "pinned", pinned), + EXTENT_DESC(NULL, "ntotal", ntotal), + EXTENT_DESC(NULL, "total", total), +}; +#undef EXTENT_DESC +_Static_assert(sizeof(stats_extent_cols) / sizeof(stats_extent_cols[0]) == + EXTENT_COL_COUNT, "stats_extent_cols must match EXTENT_COL_COUNT"); + +static const unsigned stats_extent_json_order[] = { + EXTENT_COL_NDIRTY, + EXTENT_COL_NMUZZY, + EXTENT_COL_NRETAINED, + EXTENT_COL_NPINNED, + EXTENT_COL_DIRTY, + EXTENT_COL_MUZZY, + EXTENT_COL_RETAINED, + EXTENT_COL_PINNED, +}; + +static void +stats_emit_arena_extent_row(emitter_t *emitter, emitter_row_t *table_row, + emitter_col_t *cols, const stats_arena_extent_emit_row_t *row, + size_t prev_size, bool is_gap) { + emitter_col_table_fill(stats_extent_cols, EXTENT_COL_COUNT, + STATS_COL_FLAG_NONE, cols, row); + char size_buf[48]; + stats_size_col_set(&cols[EXTENT_COL_SIZE], row->size, prev_size, + size_buf, sizeof(size_buf)); + emitter_json_object_begin(emitter); + emitter_col_table_emit_json(emitter, stats_extent_cols, EXTENT_COL_COUNT, + STATS_COL_FLAG_NONE, cols, stats_extent_json_order, + sizeof(stats_extent_json_order) / + sizeof(stats_extent_json_order[0])); + emitter_json_object_end(emitter); + emitter_table_sparse_row(emitter, table_row, is_gap); +} + JEMALLOC_COLD static void stats_arena_extents_print(emitter_t *emitter, unsigned i) { /* Streaming table; see stats_arena_bins_print for the rationale. */ unsigned j; - emitter_row_t header_row; - emitter_row_init(&header_row); - emitter_row_t row; - emitter_row_init(&row); - - COL_HDR(row, size, NULL, right, 20, size) - COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, ndirty, NULL, right, 13, size) - COL_HDR(row, dirty, NULL, right, 13, size) - COL_HDR(row, nmuzzy, NULL, right, 13, size) - COL_HDR(row, muzzy, NULL, right, 13, size) - COL_HDR(row, nretained, NULL, right, 13, size) - COL_HDR(row, retained, NULL, right, 13, size) - COL_HDR(row, npinned, NULL, right, 13, size) - COL_HDR(row, pinned, NULL, right, 13, size) - COL_HDR(row, ntotal, NULL, right, 13, size) - COL_HDR(row, total, NULL, right, 13, size) - - /* Label this section. */ - header_size.width -= 8; - emitter_table_printf(emitter, "extents:"); - emitter_table_row(emitter, &header_row); - emitter_json_array_kv_begin(emitter, "extents"); + emitter_row_t row, header_row; + emitter_col_t cols[EXTENT_COL_COUNT], header_cols[EXTENT_COL_COUNT]; + emitter_col_table_build(stats_extent_cols, EXTENT_COL_COUNT, + STATS_COL_FLAG_NONE, &row, cols, &header_row, header_cols); + emitter_col_table_header(emitter, &header_row, + &header_cols[EXTENT_COL_SIZE], "extents:", "extents"); size_t stats_arenas_mib[CTL_MAX_DEPTH]; CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas"); @@ -1093,40 +1333,9 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { bool is_gap = (e.ntotal == 0); - emitter_json_object_begin(emitter); - emitter_json_kv(emitter, "ndirty", emitter_type_size, &e.ndirty); - emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &e.nmuzzy); - emitter_json_kv( - emitter, "nretained", emitter_type_size, &e.nretained); - emitter_json_kv( - emitter, "npinned", emitter_type_size, &e.npinned); - - emitter_json_kv( - emitter, "dirty_bytes", emitter_type_size, &e.dirty_bytes); - emitter_json_kv( - emitter, "muzzy_bytes", emitter_type_size, &e.muzzy_bytes); - emitter_json_kv(emitter, "retained_bytes", emitter_type_size, - &e.retained_bytes); - emitter_json_kv(emitter, "pinned_bytes", emitter_type_size, - &e.pinned_bytes); - emitter_json_object_end(emitter); - - char size_buf[48]; - stats_size_col_set(&col_size, sz_pind2sz(j), - j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); - col_ind.unsigned_val = j; - col_ndirty.size_val = e.ndirty; - col_dirty.size_val = e.dirty_bytes; - col_nmuzzy.size_val = e.nmuzzy; - col_muzzy.size_val = e.muzzy_bytes; - col_nretained.size_val = e.nretained; - col_retained.size_val = e.retained_bytes; - col_npinned.size_val = e.npinned; - col_pinned.size_val = e.pinned_bytes; - col_ntotal.size_val = e.ntotal; - col_total.size_val = e.total_bytes; - - emitter_table_sparse_row(emitter, &row, is_gap); + stats_arena_extent_emit_row_t emit_row = {&e, j, sz_pind2sz(j)}; + stats_emit_arena_extent_row(emitter, &row, cols, &emit_row, + j > 0 ? sz_pind2sz(j - 1) : 0, is_gap); } emitter_json_array_end(emitter); /* Close "extents". */ emitter_table_sparse_end(emitter); @@ -1319,59 +1528,133 @@ stats_arena_hpa_shard_counters_print( emitter_json_array_end(emitter); /* End "alloc_batch" */ } +typedef struct { + const stats_arena_hpa_slab_t *slab; + const char *size_title; + size_t size; + size_t prev_size; + unsigned ind; +} stats_arena_hpa_slab_emit_row_t; + +#define HPA_SLAB_COL_GET(name) \ + static void \ + stats_hpa_slab_col_get_##name(const void *vrow, emitter_col_t *col) { \ + const stats_arena_hpa_slab_emit_row_t *row = vrow; \ + col->size_val = row->slab->name; \ + } +HPA_SLAB_COL_GET(npageslabs_huge) +HPA_SLAB_COL_GET(nactive_huge) +HPA_SLAB_COL_GET(ndirty_huge) +HPA_SLAB_COL_GET(npageslabs_nonhuge) +HPA_SLAB_COL_GET(nactive_nonhuge) +HPA_SLAB_COL_GET(ndirty_nonhuge) +HPA_SLAB_COL_GET(nretained_nonhuge) +#undef HPA_SLAB_COL_GET + static void -stats_emit_arena_hpa_slab_json(emitter_t *emitter, - const stats_arena_hpa_slab_t *s, const char *json_key) { - emitter_json_object_kv_begin(emitter, json_key); - emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &s->npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &s->nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &s->ndirty_huge); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &s->npageslabs_nonhuge); - emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &s->nactive_nonhuge); - emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &s->ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &s->nretained_nonhuge); - emitter_json_object_end(emitter); +stats_hpa_slab_col_get_size(const void *vrow, emitter_col_t *col) { + const stats_arena_hpa_slab_emit_row_t *row = vrow; + if (row->size_title != NULL) { + col->type = emitter_type_title; + col->str_val = row->size_title; + } else { + col->size_val = row->size; + } } -/* Fill the shared count columns of one pageslab row from s. */ static void -stats_hpa_slab_cols_set(const stats_arena_hpa_slab_t *s, - emitter_col_t *npageslabs_huge, emitter_col_t *nactive_huge, - emitter_col_t *ndirty_huge, emitter_col_t *npageslabs_nonhuge, - emitter_col_t *nactive_nonhuge, emitter_col_t *ndirty_nonhuge, - emitter_col_t *nretained_nonhuge) { - npageslabs_huge->size_val = s->npageslabs_huge; - nactive_huge->size_val = s->nactive_huge; - ndirty_huge->size_val = s->ndirty_huge; - npageslabs_nonhuge->size_val = s->npageslabs_nonhuge; - nactive_nonhuge->size_val = s->nactive_nonhuge; - ndirty_nonhuge->size_val = s->ndirty_nonhuge; - nretained_nonhuge->size_val = s->nretained_nonhuge; +stats_hpa_slab_col_get_ind(const void *vrow, emitter_col_t *col) { + const stats_arena_hpa_slab_emit_row_t *row = vrow; + if (row->size_title != NULL) { + col->type = emitter_type_title; + col->str_val = "-"; + } else { + col->unsigned_val = row->ind; + } +} + +enum { + HPA_SLAB_COL_SIZE, + HPA_SLAB_COL_IND, + HPA_SLAB_COL_NPAGESLABS_HUGE, + HPA_SLAB_COL_NACTIVE_HUGE, + HPA_SLAB_COL_NDIRTY_HUGE, + HPA_SLAB_COL_NPAGESLABS_NONHUGE, + HPA_SLAB_COL_NACTIVE_NONHUGE, + HPA_SLAB_COL_NDIRTY_NONHUGE, + HPA_SLAB_COL_NRETAINED_NONHUGE, + HPA_SLAB_COL_COUNT +}; + +#define HPA_SLAB_DESC(name, width) \ + {#name, #name, emitter_justify_right, width, emitter_type_size, \ + STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_##name} +static const emitter_col_desc_t stats_hpa_slab_cols[] = { + {NULL, "size", emitter_justify_right, 20, emitter_type_size, + STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_size}, + {NULL, "ind", emitter_justify_right, 4, emitter_type_unsigned, + STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_ind}, + HPA_SLAB_DESC(npageslabs_huge, 16), + HPA_SLAB_DESC(nactive_huge, 16), + HPA_SLAB_DESC(ndirty_huge, 16), + HPA_SLAB_DESC(npageslabs_nonhuge, 20), + HPA_SLAB_DESC(nactive_nonhuge, 20), + HPA_SLAB_DESC(ndirty_nonhuge, 20), + HPA_SLAB_DESC(nretained_nonhuge, 20), +}; +#undef HPA_SLAB_DESC +_Static_assert( + sizeof(stats_hpa_slab_cols) / sizeof(stats_hpa_slab_cols[0]) == + HPA_SLAB_COL_COUNT, + "stats_hpa_slab_cols must match HPA_SLAB_COL_COUNT"); + +static const unsigned stats_hpa_slab_json_order[] = { + HPA_SLAB_COL_NPAGESLABS_HUGE, + HPA_SLAB_COL_NACTIVE_HUGE, + HPA_SLAB_COL_NDIRTY_HUGE, + HPA_SLAB_COL_NPAGESLABS_NONHUGE, + HPA_SLAB_COL_NACTIVE_NONHUGE, + HPA_SLAB_COL_NDIRTY_NONHUGE, + HPA_SLAB_COL_NRETAINED_NONHUGE, +}; + +static void +stats_emit_arena_hpa_slab_row(emitter_t *emitter, + emitter_row_t *table_row, emitter_col_t *cols, + const stats_arena_hpa_slab_emit_row_t *row, const char *json_key, + bool sparse, bool is_gap) { + emitter_col_table_fill(stats_hpa_slab_cols, HPA_SLAB_COL_COUNT, + STATS_COL_FLAG_NONE, cols, row); + char size_buf[48]; + if (row->size_title == NULL) { + stats_size_col_set(&cols[HPA_SLAB_COL_SIZE], row->size, + row->prev_size, size_buf, sizeof(size_buf)); + } + if (json_key != NULL) { + emitter_json_object_kv_begin(emitter, json_key); + } else { + emitter_json_object_begin(emitter); + } + emitter_col_table_emit_json(emitter, stats_hpa_slab_cols, + HPA_SLAB_COL_COUNT, STATS_COL_FLAG_NONE, cols, + stats_hpa_slab_json_order, + sizeof(stats_hpa_slab_json_order) / + sizeof(stats_hpa_slab_json_order[0])); + emitter_json_object_end(emitter); + if (sparse) { + emitter_table_sparse_row(emitter, table_row, is_gap); + } else { + emitter_table_row(emitter, table_row); + } } static void stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { - emitter_row_t header_row; - emitter_row_init(&header_row); - emitter_row_t row; - emitter_row_init(&row); - - COL_HDR(row, size, NULL, right, 20, size) - COL_HDR(row, ind, NULL, right, 4, unsigned) - COL_HDR(row, npageslabs_huge, NULL, right, 16, size) - COL_HDR(row, nactive_huge, NULL, right, 16, size) - COL_HDR(row, ndirty_huge, NULL, right, 16, size) - COL_HDR(row, npageslabs_nonhuge, NULL, right, 20, size) - COL_HDR(row, nactive_nonhuge, NULL, right, 20, size) - COL_HDR(row, ndirty_nonhuge, NULL, right, 20, size) - COL_HDR(row, nretained_nonhuge, NULL, right, 20, size) + emitter_row_t row, header_row; + emitter_col_t cols[HPA_SLAB_COL_COUNT]; + emitter_col_t header_cols[HPA_SLAB_COL_COUNT]; + emitter_col_table_build(stats_hpa_slab_cols, HPA_SLAB_COL_COUNT, + STATS_COL_FLAG_NONE, &row, cols, &header_row, header_cols); emitter_table_printf(emitter, "pageslabs:\n"); emitter_table_row(emitter, &header_row); @@ -1381,31 +1664,19 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { * "full"/"empty", no size-class index) at the top of the table; their * JSON stays in the separate full_slabs / empty_slabs objects. */ - col_ind.type = emitter_type_title; - col_ind.str_val = "-"; - col_size.type = emitter_type_title; - stats_arena_hpa_slab_t sfull; stats_gather_arena_hpa_slab(i, "full_slabs", &sfull); - stats_emit_arena_hpa_slab_json(emitter, &sfull, "full_slabs"); - col_size.str_val = "full"; - stats_hpa_slab_cols_set(&sfull, &col_npageslabs_huge, &col_nactive_huge, - &col_ndirty_huge, &col_npageslabs_nonhuge, &col_nactive_nonhuge, - &col_ndirty_nonhuge, &col_nretained_nonhuge); - emitter_table_row(emitter, &row); + stats_arena_hpa_slab_emit_row_t full_row = { + &sfull, "full", 0, 0, 0}; + stats_emit_arena_hpa_slab_row(emitter, &row, cols, &full_row, + "full_slabs", false, false); stats_arena_hpa_slab_t sempty; stats_gather_arena_hpa_slab(i, "empty_slabs", &sempty); - stats_emit_arena_hpa_slab_json(emitter, &sempty, "empty_slabs"); - col_size.str_val = "empty"; - stats_hpa_slab_cols_set(&sempty, &col_npageslabs_huge, &col_nactive_huge, - &col_ndirty_huge, &col_npageslabs_nonhuge, &col_nactive_nonhuge, - &col_ndirty_nonhuge, &col_nretained_nonhuge); - emitter_table_row(emitter, &row); - - /* Numeric size/ind for the per-size-class rows below. */ - col_size.type = emitter_type_size; - col_ind.type = emitter_type_unsigned; + stats_arena_hpa_slab_emit_row_t empty_row = { + &sempty, "empty", 0, 0, 0}; + stats_emit_arena_hpa_slab_row(emitter, &row, cols, &empty_row, + "empty_slabs", false, false); size_t stats_arenas_mib[CTL_MAX_DEPTH]; CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas"); @@ -1420,35 +1691,11 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { bool is_gap = (s.npageslabs_huge == 0 && s.npageslabs_nonhuge == 0); - char size_buf[48]; - stats_size_col_set(&col_size, sz_pind2sz(j), - j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); - col_ind.unsigned_val = j; - col_npageslabs_huge.size_val = s.npageslabs_huge; - col_nactive_huge.size_val = s.nactive_huge; - col_ndirty_huge.size_val = s.ndirty_huge; - col_npageslabs_nonhuge.size_val = s.npageslabs_nonhuge; - col_nactive_nonhuge.size_val = s.nactive_nonhuge; - col_ndirty_nonhuge.size_val = s.ndirty_nonhuge; - col_nretained_nonhuge.size_val = s.nretained_nonhuge; - emitter_table_sparse_row(emitter, &row, is_gap); - - emitter_json_object_begin(emitter); - emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size, - &s.npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &s.nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &s.ndirty_huge); - emitter_json_kv(emitter, "npageslabs_nonhuge", - emitter_type_size, &s.npageslabs_nonhuge); - emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size, - &s.nactive_nonhuge); - emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size, - &s.ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &s.nretained_nonhuge); - emitter_json_object_end(emitter); + stats_arena_hpa_slab_emit_row_t emit_row = { + &s, NULL, sz_pind2sz(j), + j > 0 ? sz_pind2sz(j - 1) : 0, j}; + stats_emit_arena_hpa_slab_row(emitter, &row, cols, &emit_row, + NULL, true, is_gap); } emitter_json_array_end(emitter); /* End "nonfull_slabs" */ emitter_table_sparse_end(emitter); diff --git a/test/unit/emitter.c b/test/unit/emitter.c index f209b773..e7728563 100644 --- a/test/unit/emitter.c +++ b/test/unit/emitter.c @@ -668,6 +668,111 @@ static const char *sparse_row_table = " 2\n" " ---\n"; +typedef struct { + size_t size; + uint64_t count; + uint64_t prof_count; + uint64_t requests; +} col_table_test_row_t; + +#define COL_TABLE_TEST_GETTER(name, member, value_member) \ + static void \ + col_table_test_get_##name(const void *vrow, emitter_col_t *col) { \ + const col_table_test_row_t *row = vrow; \ + col->value_member = row->member; \ + } +COL_TABLE_TEST_GETTER(size, size, size_val) +COL_TABLE_TEST_GETTER(count, count, uint64_val) +COL_TABLE_TEST_GETTER(prof_count, prof_count, uint64_val) +COL_TABLE_TEST_GETTER(requests, requests, uint64_val) +#undef COL_TABLE_TEST_GETTER + +enum { + COL_TABLE_TEST_SIZE, + COL_TABLE_TEST_COUNT, + COL_TABLE_TEST_PROF_COUNT, + COL_TABLE_TEST_REQUESTS, + COL_TABLE_TEST_NCOLS +}; + +#define COL_TABLE_TEST_FLAG_PROF (1U << 0) + +static const emitter_col_desc_t col_table_test_descs[] = { + {NULL, "size", emitter_justify_right, 8, emitter_type_size, 0, + col_table_test_get_size}, + {"count", "count", emitter_justify_right, 8, emitter_type_uint64, + 0, col_table_test_get_count}, + {"prof_count", "prof", emitter_justify_right, 6, emitter_type_uint64, + COL_TABLE_TEST_FLAG_PROF, col_table_test_get_prof_count}, + {"requests", "requests", emitter_justify_right, 10, + emitter_type_uint64, 0, col_table_test_get_requests}, +}; +_Static_assert( + sizeof(col_table_test_descs) / sizeof(col_table_test_descs[0]) == + COL_TABLE_TEST_NCOLS, + "col_table_test_descs must match COL_TABLE_TEST_NCOLS"); + +TEST_BEGIN(test_col_desc_flags) { + expect_true(emitter_col_desc_active( + &col_table_test_descs[COL_TABLE_TEST_COUNT], 0), + "A column without requirements should always be active"); + expect_false(emitter_col_desc_active( + &col_table_test_descs[COL_TABLE_TEST_PROF_COUNT], 0), + "A column with an unmet requirement should be inactive"); + expect_true(emitter_col_desc_active( + &col_table_test_descs[COL_TABLE_TEST_PROF_COUNT], + COL_TABLE_TEST_FLAG_PROF), + "A column with a satisfied requirement should be active"); +} +TEST_END + +static const unsigned col_table_test_json_order[] = { + COL_TABLE_TEST_REQUESTS, + COL_TABLE_TEST_PROF_COUNT, + COL_TABLE_TEST_COUNT, +}; + +static void +emit_col_table(emitter_t *emitter) { + col_table_test_row_t value = {42, 7, 100, 9}; + emitter_row_t row, header_row; + emitter_col_t cols[COL_TABLE_TEST_NCOLS]; + emitter_col_t header_cols[COL_TABLE_TEST_NCOLS]; + + emitter_begin(emitter); + emitter_col_table_build(col_table_test_descs, COL_TABLE_TEST_NCOLS, + COL_TABLE_TEST_FLAG_PROF, &row, cols, &header_row, header_cols); + emitter_col_table_header( + emitter, &header_row, &header_cols[0], "mock:", "rows"); + emitter_col_table_fill(col_table_test_descs, COL_TABLE_TEST_NCOLS, + COL_TABLE_TEST_FLAG_PROF, cols, &value); + emitter_json_object_begin(emitter); + emitter_col_table_emit_json(emitter, col_table_test_descs, + COL_TABLE_TEST_NCOLS, COL_TABLE_TEST_FLAG_PROF, cols, + col_table_test_json_order, sizeof(col_table_test_json_order) / + sizeof(col_table_test_json_order[0])); + emitter_json_object_end(emitter); + emitter_table_row(emitter, &row); + emitter_json_array_end(emitter); + emitter_end(emitter); +} + +static const char *col_table_json = + "{\n" + "\t\"rows\": [\n" + "\t\t{\n" + "\t\t\t\"requests\": 9,\n" + "\t\t\t\"prof_count\": 100,\n" + "\t\t\t\"count\": 7\n" + "\t\t}\n" + "\t]\n" + "}\n"; +static const char *col_table_json_compact = + "{\"rows\":[{\"requests\":9,\"prof_count\":100,\"count\":7}]}"; +static const char *col_table_table = + "mock:size count prof requests\n" + " 42 7 100 9\n"; + #define GENERATE_TEST(feature) \ TEST_BEGIN(test_##feature) { \ expect_emit_output(emit_##feature, feature##_json, \ @@ -685,10 +790,12 @@ GENERATE_TEST(json_nested_array) GENERATE_TEST(table_row) GENERATE_TEST(row) GENERATE_TEST(sparse_row) +GENERATE_TEST(col_table) int main(void) { return test_no_reentrancy(test_dict, test_table_printf, test_nested_dict, test_types, test_modal, test_json_array, - test_json_nested_array, test_table_row, test_row, test_sparse_row); + test_json_nested_array, test_table_row, test_row, test_sparse_row, + test_col_table, test_col_desc_flags); } From 82906161ebafb3310d14e01ba5099515e9790665 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 17 Jul 2026 09:24:26 -0700 Subject: [PATCH 9/9] Expose stats table columns in JSON output and simplify code --- include/jemalloc/internal/emitter.h | 30 ++-- src/stats.c | 246 ++++++++-------------------- test/unit/emitter.c | 37 ++--- test/unit/json_stats.c | 67 +++++++- 4 files changed, 163 insertions(+), 217 deletions(-) diff --git a/include/jemalloc/internal/emitter.h b/include/jemalloc/internal/emitter.h index 359077e6..10082ec7 100644 --- a/include/jemalloc/internal/emitter.h +++ b/include/jemalloc/internal/emitter.h @@ -591,13 +591,12 @@ emitter_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) { } /* - * Descriptor-driven column tables. A descriptor array defines table order; - * callers provide a separate index array when JSON requires a different - * subset or ordering. The getter lets one engine serve different row types - * while keeping derived values (rates, utilization, and so on) with the - * table-specific code. A column is active when all its required_flags are - * present in the active_flags supplied by the caller; the emitter assigns no - * domain-specific meaning to those bits. + * Descriptor-driven column tables. A descriptor array defines table and JSON + * order; columns with a NULL json_key are table-only. The getter lets one + * engine serve different row types while keeping derived values (rates, + * utilization, and so on) with the table-specific code. A column is active + * when all its required_flags are present in the active_flags supplied by the + * caller; the emitter assigns no domain-specific meaning to those bits. */ typedef struct emitter_col_desc_s emitter_col_desc_t; struct emitter_col_desc_s { @@ -666,17 +665,18 @@ emitter_col_table_fill(const emitter_col_desc_t *descs, size_t ndescs, static inline void emitter_col_table_emit_json(emitter_t *emitter, const emitter_col_desc_t *descs, size_t ndescs, unsigned active_flags, - emitter_col_t *cols, const unsigned *json_order, size_t json_order_len) { - for (size_t i = 0; i < json_order_len; i++) { - unsigned col_ind = json_order[i]; - assert(col_ind < ndescs); - const emitter_col_desc_t *desc = &descs[col_ind]; + emitter_col_t *cols) { + for (size_t i = 0; i < ndescs; i++) { + const emitter_col_desc_t *desc = &descs[i]; if (!emitter_col_desc_active(desc, active_flags)) { continue; } - assert(desc->json_key != NULL); - emitter_json_kv(emitter, desc->json_key, cols[col_ind].type, - (const void *)&cols[col_ind].bool_val); + if (desc->json_key != NULL) { + emitter_type_t json_type = cols[i].type == emitter_type_title + ? emitter_type_string : cols[i].type; + emitter_json_kv(emitter, desc->json_key, json_type, + (const void *)&cols[i].bool_val); + } } } diff --git a/src/stats.c b/src/stats.c index 8ae8ca94..876f7ec8 100644 --- a/src/stats.c +++ b/src/stats.c @@ -782,54 +782,27 @@ stats_bin_col_get_util(const void *vrow, emitter_col_t *col) { col->str_val = row->util; } -enum { - BIN_COL_SIZE, - BIN_COL_IND, - BIN_COL_ALLOCATED, - BIN_COL_NMALLOC, - BIN_COL_NMALLOC_PS, - BIN_COL_NDALLOC, - BIN_COL_NDALLOC_PS, - BIN_COL_NREQUESTS, - BIN_COL_NREQUESTS_PS, - BIN_COL_PROF_LIVE_REQUESTED, - BIN_COL_PROF_LIVE_COUNT, - BIN_COL_PROF_ACCUM_REQUESTED, - BIN_COL_PROF_ACCUM_COUNT, - BIN_COL_NSHARDS, - BIN_COL_CURREGS, - BIN_COL_CURSLABS, - BIN_COL_NONFULL_SLABS, - BIN_COL_REGS, - BIN_COL_PGS, - BIN_COL_SPACER, - BIN_COL_UTIL, - BIN_COL_NFILLS, - BIN_COL_NFILLS_PS, - BIN_COL_NFLUSHES, - BIN_COL_NFLUSHES_PS, - BIN_COL_NSLABS, - BIN_COL_NRESLABS, - BIN_COL_NRESLABS_PS, - BIN_COL_COUNT -}; +#define BIN_COL_SIZE 0 #define BIN_DESC(key, label, width, type, flags, name) \ {key, label, emitter_justify_right, width, emitter_type_##type, flags, \ stats_bin_col_get_##name} static const emitter_col_desc_t stats_bin_cols[] = { - BIN_DESC(NULL, "size", 20, size, STATS_COL_FLAG_NONE, size), - BIN_DESC(NULL, "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind), - BIN_DESC(NULL, "allocated", 14, size, STATS_COL_FLAG_NONE, allocated), + BIN_DESC("size", "size", 20, size, STATS_COL_FLAG_NONE, size), + BIN_DESC("ind", "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind), + BIN_DESC("allocated", "allocated", 14, size, STATS_COL_FLAG_NONE, + allocated), BIN_DESC("nmalloc", "nmalloc", 14, uint64, STATS_COL_FLAG_NONE, nmalloc), - BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, nmalloc_ps), + BIN_DESC("nmalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + nmalloc_ps), BIN_DESC("ndalloc", "ndalloc", 14, uint64, STATS_COL_FLAG_NONE, ndalloc), - BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, ndalloc_ps), + BIN_DESC("ndalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + ndalloc_ps), BIN_DESC("nrequests", "nrequests", 15, uint64, STATS_COL_FLAG_NONE, nrequests), - BIN_DESC(NULL, "(#/sec)", 10, uint64, STATS_COL_FLAG_NONE, + BIN_DESC("nrequests_ps", "(#/sec)", 10, uint64, STATS_COL_FLAG_NONE, nrequests_ps), BIN_DESC("prof_live_requested", "prof_live_requested", 21, uint64, STATS_COL_FLAG_PROF, prof_live_requested), @@ -839,48 +812,33 @@ static const emitter_col_desc_t stats_bin_cols[] = { STATS_COL_FLAG_PROF, prof_accum_requested), BIN_DESC("prof_accum_count", "prof_accum_count", 17, uint64, STATS_COL_FLAG_PROF, prof_accum_count), - BIN_DESC(NULL, "nshards", 9, unsigned, STATS_COL_FLAG_NONE, nshards), + BIN_DESC("nshards", "nshards", 9, unsigned, STATS_COL_FLAG_NONE, + nshards), BIN_DESC("curregs", "curregs", 13, size, STATS_COL_FLAG_NONE, curregs), BIN_DESC("curslabs", "curslabs", 13, size, STATS_COL_FLAG_NONE, curslabs), BIN_DESC("nonfull_slabs", "nonfull_slabs", 15, size, STATS_COL_FLAG_NONE, nonfull_slabs), - BIN_DESC(NULL, "regs", 5, unsigned, STATS_COL_FLAG_NONE, regs), - BIN_DESC(NULL, "pgs", 4, size, STATS_COL_FLAG_NONE, pgs), + BIN_DESC("regs", "regs", 5, unsigned, STATS_COL_FLAG_NONE, regs), + BIN_DESC("pgs", "pgs", 4, size, STATS_COL_FLAG_NONE, pgs), BIN_DESC(NULL, " ", 1, title, STATS_COL_FLAG_NONE, spacer), - BIN_DESC(NULL, "util", 6, title, STATS_COL_FLAG_NONE, util), + BIN_DESC("util", "util", 6, title, STATS_COL_FLAG_NONE, util), BIN_DESC("nfills", "nfills", 13, uint64, STATS_COL_FLAG_NONE, nfills), - BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, nfills_ps), + BIN_DESC("nfills_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + nfills_ps), BIN_DESC("nflushes", "nflushes", 13, uint64, STATS_COL_FLAG_NONE, nflushes), - BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + BIN_DESC("nflushes_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, nflushes_ps), - BIN_DESC(NULL, "nslabs", 13, uint64, STATS_COL_FLAG_NONE, nslabs), + BIN_DESC("nslabs", "nslabs", 13, uint64, STATS_COL_FLAG_NONE, nslabs), BIN_DESC("nreslabs", "nreslabs", 13, uint64, STATS_COL_FLAG_NONE, nreslabs), - BIN_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + BIN_DESC("nreslabs_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, nreslabs_ps), }; #undef BIN_DESC -_Static_assert(sizeof(stats_bin_cols) / sizeof(stats_bin_cols[0]) == - BIN_COL_COUNT, "stats_bin_cols must match BIN_COL_COUNT"); - -static const unsigned stats_bin_json_order[] = { - BIN_COL_NMALLOC, - BIN_COL_NDALLOC, - BIN_COL_CURREGS, - BIN_COL_NREQUESTS, - BIN_COL_PROF_LIVE_REQUESTED, - BIN_COL_PROF_LIVE_COUNT, - BIN_COL_PROF_ACCUM_REQUESTED, - BIN_COL_PROF_ACCUM_COUNT, - BIN_COL_NFILLS, - BIN_COL_NFLUSHES, - BIN_COL_NRESLABS, - BIN_COL_CURSLABS, - BIN_COL_NONFULL_SLABS, -}; +#define BIN_COL_COUNT (sizeof(stats_bin_cols) / sizeof(stats_bin_cols[0])) static void stats_emit_arena_bin_row(emitter_t *emitter, emitter_row_t *table_row, @@ -892,8 +850,7 @@ stats_emit_arena_bin_row(emitter_t *emitter, emitter_row_t *table_row, stats_bin_cols, BIN_COL_COUNT, active_flags, cols, row); emitter_json_object_begin(emitter); emitter_col_table_emit_json(emitter, stats_bin_cols, BIN_COL_COUNT, - active_flags, cols, stats_bin_json_order, - sizeof(stats_bin_json_order) / sizeof(stats_bin_json_order[0])); + active_flags, cols); if (mutex) { emitter_json_object_kv_begin(emitter, "mutex"); mutex_stats_emit(emitter, NULL, mutex64, mutex32); @@ -1068,41 +1025,29 @@ stats_lextent_col_get_allocated(const void *vrow, emitter_col_t *col) { col->size_val = row->lextent->curlextents * row->lextent->lextent_size; } -enum { - LEXTENT_COL_SIZE, - LEXTENT_COL_IND, - LEXTENT_COL_ALLOCATED, - LEXTENT_COL_NMALLOC, - LEXTENT_COL_NMALLOC_PS, - LEXTENT_COL_NDALLOC, - LEXTENT_COL_NDALLOC_PS, - LEXTENT_COL_NREQUESTS, - LEXTENT_COL_NREQUESTS_PS, - LEXTENT_COL_PROF_LIVE_REQUESTED, - LEXTENT_COL_PROF_LIVE_COUNT, - LEXTENT_COL_PROF_ACCUM_REQUESTED, - LEXTENT_COL_PROF_ACCUM_COUNT, - LEXTENT_COL_CURLEXTENTS, - LEXTENT_COL_COUNT -}; +#define LEXTENT_COL_SIZE 0 #define LEXTENT_DESC(key, label, width, type, flags, name) \ {key, label, emitter_justify_right, width, emitter_type_##type, flags, \ stats_lextent_col_get_##name} static const emitter_col_desc_t stats_lextent_cols[] = { - LEXTENT_DESC(NULL, "size", 20, size, STATS_COL_FLAG_NONE, size), - LEXTENT_DESC(NULL, "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind), - LEXTENT_DESC(NULL, "allocated", 13, size, STATS_COL_FLAG_NONE, + LEXTENT_DESC("size", "size", 20, size, STATS_COL_FLAG_NONE, size), + LEXTENT_DESC("ind", "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind), + LEXTENT_DESC("allocated", "allocated", 13, size, STATS_COL_FLAG_NONE, allocated), - LEXTENT_DESC(NULL, "nmalloc", 13, uint64, STATS_COL_FLAG_NONE, nmalloc), - LEXTENT_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + LEXTENT_DESC("nmalloc", "nmalloc", 13, uint64, STATS_COL_FLAG_NONE, + nmalloc), + LEXTENT_DESC("nmalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, nmalloc_ps), - LEXTENT_DESC(NULL, "ndalloc", 13, uint64, STATS_COL_FLAG_NONE, ndalloc), - LEXTENT_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + LEXTENT_DESC("ndalloc", "ndalloc", 13, uint64, STATS_COL_FLAG_NONE, + ndalloc), + LEXTENT_DESC("ndalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, ndalloc_ps), - LEXTENT_DESC(NULL, "nrequests", 13, uint64, STATS_COL_FLAG_NONE, + LEXTENT_DESC("nrequests", "nrequests", 13, uint64, + STATS_COL_FLAG_NONE, nrequests), - LEXTENT_DESC(NULL, "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE, + LEXTENT_DESC("nrequests_ps", "(#/sec)", 8, uint64, + STATS_COL_FLAG_NONE, nrequests_ps), LEXTENT_DESC("prof_live_requested", "prof_live_requested", 21, uint64, STATS_COL_FLAG_PROF, prof_live_requested), @@ -1116,16 +1061,8 @@ static const emitter_col_desc_t stats_lextent_cols[] = { STATS_COL_FLAG_NONE, curlextents), }; #undef LEXTENT_DESC -_Static_assert(sizeof(stats_lextent_cols) / sizeof(stats_lextent_cols[0]) == - LEXTENT_COL_COUNT, "stats_lextent_cols must match LEXTENT_COL_COUNT"); - -static const unsigned stats_lextent_json_order[] = { - LEXTENT_COL_PROF_LIVE_REQUESTED, - LEXTENT_COL_PROF_LIVE_COUNT, - LEXTENT_COL_PROF_ACCUM_REQUESTED, - LEXTENT_COL_PROF_ACCUM_COUNT, - LEXTENT_COL_CURLEXTENTS, -}; +#define LEXTENT_COL_COUNT \ + (sizeof(stats_lextent_cols) / sizeof(stats_lextent_cols[0])) static void stats_emit_arena_lextent_row(emitter_t *emitter, emitter_row_t *table_row, @@ -1134,16 +1071,14 @@ stats_emit_arena_lextent_row(emitter_t *emitter, emitter_row_t *table_row, unsigned active_flags = stats_col_active_flags(prof_stats_on); emitter_col_table_fill(stats_lextent_cols, LEXTENT_COL_COUNT, active_flags, cols, row); + emitter_json_object_begin(emitter); + emitter_col_table_emit_json(emitter, stats_lextent_cols, + LEXTENT_COL_COUNT, active_flags, cols); + emitter_json_object_end(emitter); char size_buf[48]; stats_size_col_set(&cols[LEXTENT_COL_SIZE], row->lextent->lextent_size, prev_size, size_buf, sizeof(size_buf)); - emitter_json_object_begin(emitter); - emitter_col_table_emit_json(emitter, stats_lextent_cols, - LEXTENT_COL_COUNT, active_flags, cols, stats_lextent_json_order, - sizeof(stats_lextent_json_order) / - sizeof(stats_lextent_json_order[0])); emitter_table_sparse_row(emitter, table_row, is_gap); - emitter_json_object_end(emitter); } JEMALLOC_COLD @@ -1241,29 +1176,15 @@ stats_extent_col_get_ind(const void *vrow, emitter_col_t *col) { col->unsigned_val = row->ind; } -enum { - EXTENT_COL_SIZE, - EXTENT_COL_IND, - EXTENT_COL_NDIRTY, - EXTENT_COL_DIRTY, - EXTENT_COL_NMUZZY, - EXTENT_COL_MUZZY, - EXTENT_COL_NRETAINED, - EXTENT_COL_RETAINED, - EXTENT_COL_NPINNED, - EXTENT_COL_PINNED, - EXTENT_COL_NTOTAL, - EXTENT_COL_TOTAL, - EXTENT_COL_COUNT -}; +#define EXTENT_COL_SIZE 0 #define EXTENT_DESC(key, label, name) \ {key, label, emitter_justify_right, 13, emitter_type_size, \ STATS_COL_FLAG_NONE, stats_extent_col_get_##name} static const emitter_col_desc_t stats_extent_cols[] = { - {NULL, "size", emitter_justify_right, 20, emitter_type_size, + {"size", "size", emitter_justify_right, 20, emitter_type_size, STATS_COL_FLAG_NONE, stats_extent_col_get_size}, - {NULL, "ind", emitter_justify_right, 4, emitter_type_unsigned, + {"ind", "ind", emitter_justify_right, 4, emitter_type_unsigned, STATS_COL_FLAG_NONE, stats_extent_col_get_ind}, EXTENT_DESC("ndirty", "ndirty", ndirty), EXTENT_DESC("dirty_bytes", "dirty", dirty), @@ -1273,23 +1194,12 @@ static const emitter_col_desc_t stats_extent_cols[] = { EXTENT_DESC("retained_bytes", "retained", retained), EXTENT_DESC("npinned", "npinned", npinned), EXTENT_DESC("pinned_bytes", "pinned", pinned), - EXTENT_DESC(NULL, "ntotal", ntotal), - EXTENT_DESC(NULL, "total", total), + EXTENT_DESC("ntotal", "ntotal", ntotal), + EXTENT_DESC("total_bytes", "total", total), }; #undef EXTENT_DESC -_Static_assert(sizeof(stats_extent_cols) / sizeof(stats_extent_cols[0]) == - EXTENT_COL_COUNT, "stats_extent_cols must match EXTENT_COL_COUNT"); - -static const unsigned stats_extent_json_order[] = { - EXTENT_COL_NDIRTY, - EXTENT_COL_NMUZZY, - EXTENT_COL_NRETAINED, - EXTENT_COL_NPINNED, - EXTENT_COL_DIRTY, - EXTENT_COL_MUZZY, - EXTENT_COL_RETAINED, - EXTENT_COL_PINNED, -}; +#define EXTENT_COL_COUNT \ + (sizeof(stats_extent_cols) / sizeof(stats_extent_cols[0])) static void stats_emit_arena_extent_row(emitter_t *emitter, emitter_row_t *table_row, @@ -1297,15 +1207,13 @@ stats_emit_arena_extent_row(emitter_t *emitter, emitter_row_t *table_row, size_t prev_size, bool is_gap) { emitter_col_table_fill(stats_extent_cols, EXTENT_COL_COUNT, STATS_COL_FLAG_NONE, cols, row); + emitter_json_object_begin(emitter); + emitter_col_table_emit_json(emitter, stats_extent_cols, EXTENT_COL_COUNT, + STATS_COL_FLAG_NONE, cols); + emitter_json_object_end(emitter); char size_buf[48]; stats_size_col_set(&cols[EXTENT_COL_SIZE], row->size, prev_size, size_buf, sizeof(size_buf)); - emitter_json_object_begin(emitter); - emitter_col_table_emit_json(emitter, stats_extent_cols, EXTENT_COL_COUNT, - STATS_COL_FLAG_NONE, cols, stats_extent_json_order, - sizeof(stats_extent_json_order) / - sizeof(stats_extent_json_order[0])); - emitter_json_object_end(emitter); emitter_table_sparse_row(emitter, table_row, is_gap); } @@ -1573,26 +1481,15 @@ stats_hpa_slab_col_get_ind(const void *vrow, emitter_col_t *col) { } } -enum { - HPA_SLAB_COL_SIZE, - HPA_SLAB_COL_IND, - HPA_SLAB_COL_NPAGESLABS_HUGE, - HPA_SLAB_COL_NACTIVE_HUGE, - HPA_SLAB_COL_NDIRTY_HUGE, - HPA_SLAB_COL_NPAGESLABS_NONHUGE, - HPA_SLAB_COL_NACTIVE_NONHUGE, - HPA_SLAB_COL_NDIRTY_NONHUGE, - HPA_SLAB_COL_NRETAINED_NONHUGE, - HPA_SLAB_COL_COUNT -}; +#define HPA_SLAB_COL_SIZE 0 #define HPA_SLAB_DESC(name, width) \ {#name, #name, emitter_justify_right, width, emitter_type_size, \ STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_##name} static const emitter_col_desc_t stats_hpa_slab_cols[] = { - {NULL, "size", emitter_justify_right, 20, emitter_type_size, + {"size", "size", emitter_justify_right, 20, emitter_type_size, STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_size}, - {NULL, "ind", emitter_justify_right, 4, emitter_type_unsigned, + {"ind", "ind", emitter_justify_right, 4, emitter_type_unsigned, STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_ind}, HPA_SLAB_DESC(npageslabs_huge, 16), HPA_SLAB_DESC(nactive_huge, 16), @@ -1603,20 +1500,8 @@ static const emitter_col_desc_t stats_hpa_slab_cols[] = { HPA_SLAB_DESC(nretained_nonhuge, 20), }; #undef HPA_SLAB_DESC -_Static_assert( - sizeof(stats_hpa_slab_cols) / sizeof(stats_hpa_slab_cols[0]) == - HPA_SLAB_COL_COUNT, - "stats_hpa_slab_cols must match HPA_SLAB_COL_COUNT"); - -static const unsigned stats_hpa_slab_json_order[] = { - HPA_SLAB_COL_NPAGESLABS_HUGE, - HPA_SLAB_COL_NACTIVE_HUGE, - HPA_SLAB_COL_NDIRTY_HUGE, - HPA_SLAB_COL_NPAGESLABS_NONHUGE, - HPA_SLAB_COL_NACTIVE_NONHUGE, - HPA_SLAB_COL_NDIRTY_NONHUGE, - HPA_SLAB_COL_NRETAINED_NONHUGE, -}; +#define HPA_SLAB_COL_COUNT \ + (sizeof(stats_hpa_slab_cols) / sizeof(stats_hpa_slab_cols[0])) static void stats_emit_arena_hpa_slab_row(emitter_t *emitter, @@ -1625,22 +1510,19 @@ stats_emit_arena_hpa_slab_row(emitter_t *emitter, bool sparse, bool is_gap) { emitter_col_table_fill(stats_hpa_slab_cols, HPA_SLAB_COL_COUNT, STATS_COL_FLAG_NONE, cols, row); - char size_buf[48]; - if (row->size_title == NULL) { - stats_size_col_set(&cols[HPA_SLAB_COL_SIZE], row->size, - row->prev_size, size_buf, sizeof(size_buf)); - } if (json_key != NULL) { emitter_json_object_kv_begin(emitter, json_key); } else { emitter_json_object_begin(emitter); } emitter_col_table_emit_json(emitter, stats_hpa_slab_cols, - HPA_SLAB_COL_COUNT, STATS_COL_FLAG_NONE, cols, - stats_hpa_slab_json_order, - sizeof(stats_hpa_slab_json_order) / - sizeof(stats_hpa_slab_json_order[0])); + HPA_SLAB_COL_COUNT, STATS_COL_FLAG_NONE, cols); emitter_json_object_end(emitter); + char size_buf[48]; + if (row->size_title == NULL) { + stats_size_col_set(&cols[HPA_SLAB_COL_SIZE], row->size, + row->prev_size, size_buf, sizeof(size_buf)); + } if (sparse) { emitter_table_sparse_row(emitter, table_row, is_gap); } else { diff --git a/test/unit/emitter.c b/test/unit/emitter.c index e7728563..3cfcf5d5 100644 --- a/test/unit/emitter.c +++ b/test/unit/emitter.c @@ -669,10 +669,11 @@ static const char *sparse_row_table = " ---\n"; typedef struct { - size_t size; - uint64_t count; - uint64_t prof_count; - uint64_t requests; + size_t size; + uint64_t count; + uint64_t prof_count; + const char *label; + uint64_t requests; } col_table_test_row_t; #define COL_TABLE_TEST_GETTER(name, member, value_member) \ @@ -684,6 +685,7 @@ typedef struct { COL_TABLE_TEST_GETTER(size, size, size_val) COL_TABLE_TEST_GETTER(count, count, uint64_val) COL_TABLE_TEST_GETTER(prof_count, prof_count, uint64_val) +COL_TABLE_TEST_GETTER(label, label, str_val) COL_TABLE_TEST_GETTER(requests, requests, uint64_val) #undef COL_TABLE_TEST_GETTER @@ -691,6 +693,7 @@ enum { COL_TABLE_TEST_SIZE, COL_TABLE_TEST_COUNT, COL_TABLE_TEST_PROF_COUNT, + COL_TABLE_TEST_LABEL, COL_TABLE_TEST_REQUESTS, COL_TABLE_TEST_NCOLS }; @@ -704,6 +707,8 @@ static const emitter_col_desc_t col_table_test_descs[] = { 0, col_table_test_get_count}, {"prof_count", "prof", emitter_justify_right, 6, emitter_type_uint64, COL_TABLE_TEST_FLAG_PROF, col_table_test_get_prof_count}, + {"label", "label", emitter_justify_right, 7, emitter_type_title, 0, + col_table_test_get_label}, {"requests", "requests", emitter_justify_right, 10, emitter_type_uint64, 0, col_table_test_get_requests}, }; @@ -726,15 +731,9 @@ TEST_BEGIN(test_col_desc_flags) { } TEST_END -static const unsigned col_table_test_json_order[] = { - COL_TABLE_TEST_REQUESTS, - COL_TABLE_TEST_PROF_COUNT, - COL_TABLE_TEST_COUNT, -}; - static void emit_col_table(emitter_t *emitter) { - col_table_test_row_t value = {42, 7, 100, 9}; + col_table_test_row_t value = {42, 7, 100, "row", 9}; emitter_row_t row, header_row; emitter_col_t cols[COL_TABLE_TEST_NCOLS]; emitter_col_t header_cols[COL_TABLE_TEST_NCOLS]; @@ -748,9 +747,7 @@ emit_col_table(emitter_t *emitter) { COL_TABLE_TEST_FLAG_PROF, cols, &value); emitter_json_object_begin(emitter); emitter_col_table_emit_json(emitter, col_table_test_descs, - COL_TABLE_TEST_NCOLS, COL_TABLE_TEST_FLAG_PROF, cols, - col_table_test_json_order, sizeof(col_table_test_json_order) / - sizeof(col_table_test_json_order[0])); + COL_TABLE_TEST_NCOLS, COL_TABLE_TEST_FLAG_PROF, cols); emitter_json_object_end(emitter); emitter_table_row(emitter, &row); emitter_json_array_end(emitter); @@ -761,17 +758,19 @@ static const char *col_table_json = "{\n" "\t\"rows\": [\n" "\t\t{\n" - "\t\t\t\"requests\": 9,\n" + "\t\t\t\"count\": 7,\n" "\t\t\t\"prof_count\": 100,\n" - "\t\t\t\"count\": 7\n" + "\t\t\t\"label\": \"row\",\n" + "\t\t\t\"requests\": 9\n" "\t\t}\n" "\t]\n" "}\n"; static const char *col_table_json_compact = - "{\"rows\":[{\"requests\":9,\"prof_count\":100,\"count\":7}]}"; + "{\"rows\":[{\"count\":7,\"prof_count\":100,\"label\":\"row\"," + "\"requests\":9}]}"; static const char *col_table_table = - "mock:size count prof requests\n" - " 42 7 100 9\n"; + "mock:size count prof label requests\n" + " 42 7 100 row 9\n"; #define GENERATE_TEST(feature) \ TEST_BEGIN(test_##feature) { \ diff --git a/test/unit/json_stats.c b/test/unit/json_stats.c index 11b397d9..8a3ac621 100644 --- a/test/unit/json_stats.c +++ b/test/unit/json_stats.c @@ -288,6 +288,70 @@ json_find_named_array( return array_begin; } +static void +expect_json_keys_before(const char *begin, const char *end, + const char *const *keys, size_t nkeys, const char *section_name) { + for (size_t i = 0; i < nkeys; i++) { + char search_key[128]; + size_t written = malloc_snprintf( + search_key, sizeof(search_key), "\"%s\":", keys[i]); + expect_zu_lt(written, sizeof(search_key), "JSON key is too long"); + const char *found = strstr(begin, search_key); + expect_true(found != NULL && found < end, + "%s should contain %s", section_name, keys[i]); + } +} + +TEST_BEGIN(test_json_stats_table_columns) { + test_skip_if(!config_stats); + + uint64_t epoch = 1; + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), + 0, "Unexpected mallctl() failure"); + + stats_buf_t sbuf; + stats_buf_init(&sbuf); + malloc_stats_print(stats_buf_write_cb, &sbuf, "Ja"); + + const char *bins_end = NULL; + const char *bins = json_find_named_array(sbuf.buf, "bins", &bins_end); + expect_ptr_not_null(bins, "JSON output should contain bins"); + const char *bin_keys[] = {"size", "ind", "allocated", "nmalloc_ps", + "nshards", "regs", "pgs", "util", "nslabs"}; + expect_json_keys_before(bins, bins_end, bin_keys, + sizeof(bin_keys) / sizeof(bin_keys[0]), "bins"); + + const char *lextents_end = NULL; + const char *lextents = json_find_named_array( + sbuf.buf, "lextents", &lextents_end); + expect_ptr_not_null(lextents, "JSON output should contain lextents"); + const char *lextent_keys[] = {"size", "ind", "allocated", "nmalloc", + "nmalloc_ps", "ndalloc", "ndalloc_ps", "nrequests", + "nrequests_ps"}; + expect_json_keys_before(lextents, lextents_end, lextent_keys, + sizeof(lextent_keys) / sizeof(lextent_keys[0]), "lextents"); + + const char *extents_end = NULL; + const char *extents = json_find_named_array( + sbuf.buf, "extents", &extents_end); + expect_ptr_not_null(extents, "JSON output should contain extents"); + const char *extent_keys[] = {"size", "ind", "ntotal", "total_bytes"}; + expect_json_keys_before(extents, extents_end, extent_keys, + sizeof(extent_keys) / sizeof(extent_keys[0]), "extents"); + + const char *full_slabs_end = NULL; + const char *full_slabs = json_find_named_object( + sbuf.buf, "full_slabs", &full_slabs_end); + expect_ptr_not_null( + full_slabs, "JSON output should contain full_slabs"); + const char *hpa_keys[] = {"size", "ind"}; + expect_json_keys_before(full_slabs, full_slabs_end, hpa_keys, + sizeof(hpa_keys) / sizeof(hpa_keys[0]), "full_slabs"); + + stats_buf_fini(&sbuf); +} +TEST_END + TEST_BEGIN(test_json_stats_mutexes) { test_skip_if(!config_stats); @@ -502,7 +566,8 @@ TEST_END int main(void) { - return test_no_reentrancy(test_json_stats_mutexes, + return test_no_reentrancy(test_json_stats_table_columns, + test_json_stats_mutexes, test_hpa_shard_json_ndirty_huge, test_hpa_shard_json_contains_sec_stats, test_hpa_shard_json_contains_retained_stats);