diff --git a/include/jemalloc/internal/emitter.h b/include/jemalloc/internal/emitter.h index a4073e6a..9e645106 100644 --- a/include/jemalloc/internal/emitter.h +++ b/include/jemalloc/internal/emitter.h @@ -58,6 +58,11 @@ struct emitter_col_s { ssize_t ssize_val; const char *str_val; }; + /* + * Optional JSON key. When set, emitter_row() emits this column as a + * "json_key": value pair in JSON mode; columns left NULL are table-only. + */ + const char *json_key; /* Filled in by initialization. */ ql_elm(emitter_col_t) link; @@ -79,6 +84,8 @@ struct emitter_s { bool item_at_depth; /* True if we emitted a key and will emit corresponding value next. */ bool emitted_key; + /* Gap-collapsing state for the emitter_table_sparse_row() bracket. */ + bool table_sparse_in_gap; }; static inline bool @@ -270,6 +277,7 @@ emitter_init(emitter_t *emitter, emitter_output_t emitter_output, emitter->item_at_depth = false; emitter->emitted_key = false; emitter->nesting_depth = 0; + emitter->table_sparse_in_gap = false; } /******************************************************************************/ @@ -447,6 +455,46 @@ static inline void emitter_col_init(emitter_col_t *col, emitter_row_t *row) { ql_elm_new(col, link); ql_tail_insert(&row->cols, col, link); + col->json_key = NULL; +} + +/* + * Sparse table rows. Some tables have one row per size class but most rows + * are all-zero; in table output those runs are collapsed to a single "---" + * separator. Bracket such a table with emitter_table_sparse_begin() / + * _end() and emit each row with emitter_table_sparse_row(), passing whether + * that row is a gap (all-zero). The emitter prints the separator on a + * gap->non-gap transition and at a trailing gap, and suppresses the gap rows + * themselves; callers no longer track gap state. Table output only -- a + * no-op in JSON, where callers emit every row's object separately. + */ +static inline void +emitter_table_sparse_begin(emitter_t *emitter) { + emitter->table_sparse_in_gap = false; +} + +static inline void +emitter_table_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) { + if (emitter->output != emitter_output_table) { + return; + } + bool prev_in_gap = emitter->table_sparse_in_gap; + emitter->table_sparse_in_gap = is_gap; + if (prev_in_gap && !is_gap) { + emitter_printf(emitter, " ---\n"); + } + if (!is_gap) { + emitter_table_row(emitter, row); + } +} + +static inline void +emitter_table_sparse_end(emitter_t *emitter) { + if (emitter->output == emitter_output_table + && emitter->table_sparse_in_gap) { + emitter_printf(emitter, " ---\n"); + } + emitter->table_sparse_in_gap = false; } /******************************************************************************/ @@ -499,6 +547,49 @@ emitter_dict_end(emitter_t *emitter) { } } +/* Internal: emit a row's json-keyed columns as JSON kvs, in column order. */ +static inline void +emitter_row_json(emitter_t *emitter, emitter_row_t *row) { + emitter_col_t *col; + ql_foreach (col, &row->cols, link) { + if (col->json_key != NULL) { + emitter_json_kv(emitter, col->json_key, col->type, + (const void *)&col->bool_val); + } + } +} + +/* + * Emit a row of columns to whichever format is active. In table mode, prints + * the aligned table row (see emitter_table_row). In JSON mode, emits each + * column that has a non-NULL json_key as a "json_key": value pair, in column + * order (columns without a json_key are table-only). This lets a caller + * describe a row once and render it to either format. For JSON objects (one + * per row), wrap the call in emitter_json_object_begin()/_end(). + */ +static inline void +emitter_row(emitter_t *emitter, emitter_row_t *row) { + emitter_table_row(emitter, row); + if (emitter_outputs_json(emitter)) { + emitter_row_json(emitter, row); + } +} + +/* + * Sparse counterpart of emitter_row() for per-size-class tables: the table + * side collapses all-zero ("gap") rows (see emitter_table_sparse_row), while + * the JSON side still emits every row's json-keyed columns. Bracket the loop + * with emitter_table_sparse_begin()/_end(), and wrap each row's JSON in + * emitter_json_object_begin()/_end() when the rows form an array of objects. + */ +static inline void +emitter_sparse_row(emitter_t *emitter, emitter_row_t *row, bool is_gap) { + emitter_table_sparse_row(emitter, row, is_gap); + if (emitter_outputs_json(emitter)) { + emitter_row_json(emitter, row); + } +} + static inline void emitter_begin(emitter_t *emitter) { if (emitter_outputs_json(emitter)) { diff --git a/include/jemalloc/internal/stats_internal.h b/include/jemalloc/internal/stats_internal.h new file mode 100644 index 00000000..0c3f7b2b --- /dev/null +++ b/include/jemalloc/internal/stats_internal.h @@ -0,0 +1,340 @@ +#ifndef JEMALLOC_INTERNAL_STATS_INTERNAL_H +#define JEMALLOC_INTERNAL_STATS_INTERNAL_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/arena.h" /* ARENA_NAME_LEN */ +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/emitter.h" +#include "jemalloc/internal/prof_stats.h" + +/* + * Internal implementation support for src/stats.c only -- do NOT include from + * other translation units. + * + * This header holds noisy local scaffolding for malloc_stats_print(): ctl + * gather plumbing, gather structs, emitter table-column helpers, and option + * list helpers. It is deliberately not a reusable stats-gather API. + */ + +#define CTL_GET(n, v, t) \ + do { \ + size_t sz = sizeof(t); \ + xmallctl(n, (void *)v, &sz, NULL, 0); \ + } while (0) + +#define CTL_LEAF_PREPARE(mib, miblen, name) \ + do { \ + assert(miblen < CTL_MAX_DEPTH); \ + size_t miblen_new = CTL_MAX_DEPTH; \ + xmallctlmibnametomib(mib, miblen, name, &miblen_new); \ + assert(miblen_new > miblen); \ + } while (0) + +#define CTL_LEAF(mib, miblen, leaf, v, t) \ + do { \ + assert(miblen < CTL_MAX_DEPTH); \ + size_t miblen_new = CTL_MAX_DEPTH; \ + size_t sz = sizeof(t); \ + xmallctlbymibname( \ + mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0); \ + assert(miblen_new == miblen + 1); \ + } while (0) + +#define CTL_MIB_GET(n, i, v, t, ind) \ + do { \ + size_t mib[CTL_MAX_DEPTH]; \ + size_t miblen = sizeof(mib) / sizeof(size_t); \ + size_t sz = sizeof(t); \ + xmallctlnametomib(n, mib, &miblen); \ + mib[(ind)] = (i); \ + xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ + } while (0) + +#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1) +#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2) + +/******************************************************************************/ +/* + * Per-section gather structs. The stats_gather_*() functions that fill them + * are static in src/stats.c. + */ + +/* One arena's memory-accounting stats (stats.arenas..*). */ +typedef struct stats_arena_mem_s { + size_t mapped; + size_t retained; + size_t pinned; + size_t base; + size_t internal; + size_t metadata_edata; + size_t metadata_rtree; + size_t metadata_thp; + size_t tcache_bytes; + size_t tcache_stashed_bytes; + size_t resident; + size_t abandoned_vm; + size_t extent_avail; + size_t page; /* arenas.page, for the active-bytes conversion */ +} stats_arena_mem_t; + +/* One arena extents row (stats.arenas..extents..*). */ +typedef struct stats_arena_extent_s { + size_t ndirty; + size_t nmuzzy; + size_t nretained; + size_t npinned; + size_t ntotal; + size_t dirty_bytes; + size_t muzzy_bytes; + size_t retained_bytes; + size_t pinned_bytes; + size_t total_bytes; +} stats_arena_extent_t; + +/* One arena's "basics" (stats.arenas..{nthreads,uptime,dss}). */ +typedef struct stats_arena_basics_s { + unsigned nthreads; + uint64_t uptime; + const char *dss; + char name[ARENA_NAME_LEN]; + bool has_name; /* false for the merged/destroyed pseudo-arenas */ +} stats_arena_basics_t; + +/* One arena's decay / page stats (stats.arenas..*). */ +typedef struct stats_arena_decay_s { + ssize_t dirty_decay_ms; + ssize_t muzzy_decay_ms; + size_t pactive; + size_t pdirty; + size_t pmuzzy; + uint64_t dirty_npurge; + uint64_t dirty_nmadvise; + uint64_t dirty_purged; + uint64_t muzzy_npurge; + uint64_t muzzy_nmadvise; + uint64_t muzzy_purged; +} stats_arena_decay_t; + +/* One size class of arena allocation counts (stats.arenas..{small,large}.*). */ +typedef struct stats_arena_alloc_s { + size_t allocated; + uint64_t nmalloc; + uint64_t ndalloc; + uint64_t nrequests; + uint64_t nfills; + uint64_t nflushes; +} stats_arena_alloc_t; + +/* + * One arena bin row (stats.arenas..bins..* + arenas.bin..*, + * plus prof.stats.bins..{live,accum} when profiling stats are enabled). + */ +typedef struct stats_arena_bin_s { + uint64_t nslabs; + uint64_t nmalloc; + uint64_t ndalloc; + uint64_t nrequests; + uint64_t nfills; + uint64_t nflushes; + uint64_t nreslabs; + size_t reg_size; + size_t slab_size; + size_t curregs; + size_t curslabs; + size_t nonfull_slabs; + uint32_t nregs; + uint32_t nshards; + prof_stats_t prof_live; + prof_stats_t prof_accum; +} stats_arena_bin_t; + +/* + * One arena large-extent row (stats.arenas..lextents..* + + * arenas.lextent..*, plus prof.stats.lextents..{live,accum} when + * profiling stats are enabled). + */ +typedef struct stats_arena_lextent_s { + uint64_t nmalloc; + uint64_t ndalloc; + uint64_t nrequests; + size_t lextent_size; + size_t curlextents; + prof_stats_t prof_live; + prof_stats_t prof_accum; +} stats_arena_lextent_t; + +/* HPA shard small-extent cache (stats.arenas..hpa_sec_*). */ +typedef struct stats_arena_hpa_sec_s { + size_t sec_bytes; + size_t sec_hits; + size_t sec_misses; + size_t sec_dalloc_flush; + size_t sec_dalloc_noflush; + size_t sec_overfills; +} stats_arena_hpa_sec_t; + +/* PAC small-extent cache (stats.arenas..pac_sec_*). */ +typedef struct stats_arena_pac_sec_s { + size_t sec_bytes; + size_t sec_hits; + size_t sec_misses; + size_t sec_dalloc_flush; + size_t sec_dalloc_noflush; +} stats_arena_pac_sec_t; + +/* HPA shard counters (stats.arenas..hpa_shard.*). */ +typedef struct stats_arena_hpa_counters_s { + size_t npageslabs; + size_t nactive; + size_t ndirty; + size_t npageslabs_nonhuge; + size_t nactive_nonhuge; + size_t ndirty_nonhuge; + size_t nretained_nonhuge; + size_t npageslabs_huge; + size_t nactive_huge; + size_t ndirty_huge; + uint64_t npurge_passes; + uint64_t npurges; + uint64_t nhugifies; + uint64_t nhugify_failures; + uint64_t ndehugifies; +} stats_arena_hpa_counters_t; + +/* One HPA slab class (full/empty/nonfull) row. */ +typedef struct stats_arena_hpa_slab_s { + size_t npageslabs_huge; + size_t nactive_huge; + size_t ndirty_huge; + size_t npageslabs_nonhuge; + size_t nactive_nonhuge; + size_t ndirty_nonhuge; + size_t nretained_nonhuge; +} stats_arena_hpa_slab_t; + +/* Arena configuration / size-class scalars (arenas.*). */ +typedef struct stats_arena_config_s { + unsigned narenas; + ssize_t dirty_decay_ms; + ssize_t muzzy_decay_ms; + size_t quantum; + size_t page; + size_t hugepage; + bool have_tcache_max; + size_t tcache_max; + unsigned nbins; + unsigned nhbins; + unsigned nlextents; +} stats_arena_config_t; + +/* Per-size-class geometry (emitted in JSON only). */ +typedef struct stats_arena_bin_meta_s { + size_t size; + uint32_t nregs; + size_t slab_size; + uint32_t nshards; +} stats_arena_bin_meta_t; + +typedef struct stats_arena_lextent_meta_s { + size_t size; +} stats_arena_lextent_meta_t; + +/* Process-wide global stats (stats.* + stats.background_thread.*). */ +typedef struct stats_global_s { + size_t allocated; + size_t active; + size_t metadata; + size_t metadata_edata; + size_t metadata_rtree; + size_t metadata_thp; + size_t resident; + size_t mapped; + size_t retained; + size_t pinned; + size_t zero_reallocs; + size_t num_background_threads; + uint64_t background_thread_num_runs; + uint64_t background_thread_run_interval; +} stats_global_t; + +/******************************************************************************/ +/* + * Emitter table-column helper macros: declare an emitter_col_t (and, for + * COL_HDR, its header column) and wire it into a row. + */ + +#define COL_DECLARE(column_name) emitter_col_t col_##column_name; + +#define COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_init(&col_##column_name, &row_name); \ + col_##column_name.justify = emitter_justify_##left_or_right; \ + col_##column_name.width = col_width; \ + col_##column_name.type = emitter_type_##etype; + +#define COL(row_name, column_name, left_or_right, col_width, etype) \ + COL_DECLARE(column_name); \ + COL_INIT(row_name, column_name, left_or_right, col_width, etype) + +#define COL_HDR_DECLARE(column_name) \ + COL_DECLARE(column_name); \ + emitter_col_t header_##column_name; + +#define COL_HDR_INIT( \ + row_name, column_name, human, left_or_right, col_width, etype) \ + COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_init(&header_##column_name, &header_##row_name); \ + header_##column_name.justify = emitter_justify_##left_or_right; \ + header_##column_name.width = col_width; \ + header_##column_name.type = emitter_type_title; \ + header_##column_name.str_val = human ? human : #column_name; + +#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ + COL_HDR_DECLARE(column_name) \ + COL_HDR_INIT( \ + row_name, column_name, human, left_or_right, col_width, etype) + +/* + * stats_general_print() config/option list helpers. Unhygienic by design: + * they assume `emitter` and the standard scratch locals are in scope at the + * call site -- bv/bsz (bool), cpv/cpsz (const char *), uv/usz (unsigned), + * i64v/i64sz, u64v/u64sz, sv/ssz (size_t), ssv/sssz (ssize_t), plus bv2/ssv2 + * for the *_MUTABLE variants. (CONFIG_WRITE_BOOL uses CTL_GET above; the + * OPT_WRITE_* variants use je_mallctl directly.) + */ +#define CONFIG_WRITE_BOOL(name) \ + do { \ + CTL_GET("config." #name, &bv, bool); \ + emitter_kv( \ + emitter, #name, "config." #name, emitter_type_bool, &bv); \ + } while (0) + +#define OPT_WRITE(name, var, size, emitter_type) \ + if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \ + emitter_kv(emitter, name, "opt." name, emitter_type, &var); \ + } + +#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname) \ + if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0 \ + && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) { \ + emitter_kv_note(emitter, name, "opt." name, emitter_type, \ + &var1, altname, emitter_type, &var2); \ + } + +#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool) +#define OPT_WRITE_BOOL_MUTABLE(name, altname) \ + OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname) + +#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned) + +#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64) +#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64) + +#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size) +#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize) +#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ + OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname) + +#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string) + +#endif /* JEMALLOC_INTERNAL_STATS_INTERNAL_H */ diff --git a/src/stats.c b/src/stats.c index d4b0f4e6..3a8b5263 100644 --- a/src/stats.c +++ b/src/stats.c @@ -10,6 +10,7 @@ #include "jemalloc/internal/prof.h" #include "jemalloc/internal/prof_inlines.h" #include "jemalloc/internal/prof_stats.h" +#include "jemalloc/internal/stats_internal.h" #include "jemalloc/internal/tcache.h" static const char *const global_mutex_names[mutex_prof_num_global_mutexes] = { @@ -24,43 +25,6 @@ static const char *const arena_mutex_names[mutex_prof_num_arena_mutexes] = { #undef OP }; -#define CTL_GET(n, v, t) \ - do { \ - size_t sz = sizeof(t); \ - xmallctl(n, (void *)v, &sz, NULL, 0); \ - } while (0) - -#define CTL_LEAF_PREPARE(mib, miblen, name) \ - do { \ - assert(miblen < CTL_MAX_DEPTH); \ - size_t miblen_new = CTL_MAX_DEPTH; \ - xmallctlmibnametomib(mib, miblen, name, &miblen_new); \ - assert(miblen_new > miblen); \ - } while (0) - -#define CTL_LEAF(mib, miblen, leaf, v, t) \ - do { \ - assert(miblen < CTL_MAX_DEPTH); \ - size_t miblen_new = CTL_MAX_DEPTH; \ - size_t sz = sizeof(t); \ - xmallctlbymibname( \ - mib, miblen, leaf, &miblen_new, (void *)v, &sz, NULL, 0); \ - assert(miblen_new == miblen + 1); \ - } while (0) - -#define CTL_MIB_GET(n, i, v, t, ind) \ - do { \ - size_t mib[CTL_MAX_DEPTH]; \ - size_t miblen = sizeof(mib) / sizeof(size_t); \ - size_t sz = sizeof(t); \ - xmallctlnametomib(n, mib, &miblen); \ - mib[(ind)] = (i); \ - xmallctlbymib(mib, miblen, (void *)v, &sz, NULL, 0); \ - } while (0) - -#define CTL_M1_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 1) -#define CTL_M2_GET(n, i, v, t) CTL_MIB_GET(n, i, v, t, 2) - /******************************************************************************/ /* Data. */ @@ -74,6 +38,471 @@ static counter_accum_t stats_interval_accumulated; /* Per thread batch accum size for stats_interval. */ uint64_t stats_interval_accum_batch; +/* + * Forward declarations for the two top-level section helpers. The public + * stats_print() appears first (just below) and calls these; their definitions + * follow it, so the file reads top-down. + */ +static void stats_general_print(emitter_t *emitter, bool omit_size_class_meta); +static void stats_print_runtime_stats(emitter_t *emitter, bool merged, + bool destroyed, + bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa); + +/******************************************************************************/ +/* + * malloc_stats_print() output structure (produced by stats_print(), below); + * each line names the function that prints it, plus its JSON key where useful. + * Quoted names are literal keys below the "jemalloc" root unless shown with a + * leading dot. + * + * ___ Begin jemalloc statistics ___ + * { "jemalloc": + * general [g] -> stats_general_print + * version -> stats_general_version + * config -> stats_general_config + * system -> stats_general_system + * opt.* -> stats_general_opts + * prof -> stats_general_prof + * arenas metadata -> stats_general_arenas_print ("arenas") + * stats (config_stats) -> stats_print_runtime_stats + * global totals -> stats_print_globals ("stats") + * background_thread -> stats_print_globals + * mutexes [x] -> stats_global_mutexes_print + * per arena [m]/[d]/[a] -> stats_arena_print ("stats.arenas") + * basics -> stats_arena_basics_print + * decay -> stats_arena_decay_print + * alloc counts -> stats_arena_alloc_print (.small/.large) + * memory -> stats_arena_mem_print + * pac/sec -> stats_arena_pac_sec_print + * mutexes [x] -> stats_arena_mutexes_print (.mutexes) + * bins [b] -> stats_arena_bins_print (.bins[j]) + * lextents [l] -> stats_arena_lextents_print (.lextents[j]) + * extents [e] -> stats_arena_extents_print (.extents[j]) + * hpa shard [h] -> stats_arena_hpa_shard_print (.hpa_shard) + * } + * --- End jemalloc statistics --- + * + * Most refactored sections read values through mallctl into typed structs + * before rendering them through the emitter. This makes their coverage easier + * to audit, but does not guarantee parity between formats: format-specific + * emission remains, including JSON-only geometry and table-only derived rates + * and summary rows. + * + * Mallctl names identify data sources, not mechanical JSON paths. For example, + * stats_gather_arena_alloc reads "stats.arenas.0.small.allocated" (replacing + * the MIB arena index at runtime), while the emitter writes it below + * jemalloc["stats.arenas"][].small.allocated. In particular, "stats" + * and "stats.arenas" are sibling keys, not nested objects. + */ + +void +stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { + int err; + uint64_t epoch; + size_t u64sz; +#define OPTION(o, v, d, s) bool v = d; + STATS_PRINT_OPTIONS +#undef OPTION + + /* + * Refresh stats, in case mallctl() was called by the application. + * + * Check for OOM here, since refreshing the ctl cache can trigger + * allocation. In practice, none of the subsequent mallctl()-related + * calls in this function will cause OOM if this one succeeds. + * */ + epoch = 1; + u64sz = sizeof(uint64_t); + err = je_mallctl( + "epoch", (void *)&epoch, &u64sz, (void *)&epoch, sizeof(uint64_t)); + if (err != 0) { + if (err == EAGAIN) { + malloc_write( + ": Memory allocation failure in " + "mallctl(\"epoch\", ...)\n"); + return; + } + malloc_write( + ": Failure in mallctl(\"epoch\", " + "...)\n"); + abort(); + } + + if (opts != NULL) { + for (unsigned i = 0; opts[i] != '\0'; i++) { + switch (opts[i]) { +#define OPTION(o, v, d, s) \ + case o: \ + v = s; \ + break; + STATS_PRINT_OPTIONS +#undef OPTION + default:; + } + } + } + + emitter_t emitter; + emitter_init(&emitter, + json ? emitter_output_json_compact : emitter_output_table, write_cb, + cbopaque); + emitter_begin(&emitter); + emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); + emitter_json_object_kv_begin(&emitter, "jemalloc"); + + if (general) { + /* + * Private (non-public) omission: the per-size-class geometry + * arrays are shown only in JSON; the raw format never did. + */ + bool omit_size_class_meta = !json; + stats_general_print(&emitter, omit_size_class_meta); + } + if (config_stats) { + stats_print_runtime_stats(&emitter, merged, destroyed, unmerged, + bins, large, mutex, extents, hpa); + } + + emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ + emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); + emitter_end(&emitter); +} + +/******************************************************************************/ +/* Statistics gathering. */ +/* + * Read stats out of the allocator through mallctl into typed per-section + * structs. Keeping the mallctl reads together (and separate from emission) + * makes coverage auditable. They are static to src/stats.c; the gather structs + * and plumbing macros they use live in stats_internal.h. + */ + +/* mib must be prepared through "stats.arenas..extents" (depth 4). */ +static void +stats_gather_arena_extent(size_t *mib, unsigned j, stats_arena_extent_t *e) { + mib[4] = j; + CTL_LEAF(mib, 5, "ndirty", &e->ndirty, size_t); + CTL_LEAF(mib, 5, "nmuzzy", &e->nmuzzy, size_t); + CTL_LEAF(mib, 5, "nretained", &e->nretained, size_t); + CTL_LEAF(mib, 5, "npinned", &e->npinned, size_t); + CTL_LEAF(mib, 5, "dirty_bytes", &e->dirty_bytes, size_t); + CTL_LEAF(mib, 5, "muzzy_bytes", &e->muzzy_bytes, size_t); + CTL_LEAF(mib, 5, "retained_bytes", &e->retained_bytes, size_t); + CTL_LEAF(mib, 5, "pinned_bytes", &e->pinned_bytes, size_t); + e->ntotal = e->ndirty + e->nmuzzy + e->nretained + e->npinned; + e->total_bytes = e->dirty_bytes + e->muzzy_bytes + e->retained_bytes + + e->pinned_bytes; +} + +static void +stats_gather_arena_mem(unsigned i, stats_arena_mem_t *mem) { + CTL_M2_GET("stats.arenas.0.mapped", i, &mem->mapped, size_t); + CTL_M2_GET("stats.arenas.0.retained", i, &mem->retained, size_t); + CTL_M2_GET("stats.arenas.0.pinned", i, &mem->pinned, size_t); + CTL_M2_GET("stats.arenas.0.base", i, &mem->base, size_t); + CTL_M2_GET("stats.arenas.0.internal", i, &mem->internal, size_t); + CTL_M2_GET("stats.arenas.0.metadata_edata", i, &mem->metadata_edata, + size_t); + CTL_M2_GET("stats.arenas.0.metadata_rtree", i, &mem->metadata_rtree, + size_t); + CTL_M2_GET("stats.arenas.0.metadata_thp", i, &mem->metadata_thp, + size_t); + CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &mem->tcache_bytes, + size_t); + CTL_M2_GET("stats.arenas.0.tcache_stashed_bytes", i, + &mem->tcache_stashed_bytes, size_t); + CTL_M2_GET("stats.arenas.0.resident", i, &mem->resident, size_t); + CTL_M2_GET("stats.arenas.0.abandoned_vm", i, &mem->abandoned_vm, + size_t); + CTL_M2_GET("stats.arenas.0.extent_avail", i, &mem->extent_avail, + size_t); + CTL_GET("arenas.page", &mem->page, size_t); +} + +static void +stats_gather_arena_basics(unsigned i, stats_arena_basics_t *b) { + CTL_M2_GET("stats.arenas.0.nthreads", i, &b->nthreads, unsigned); + CTL_M2_GET("stats.arenas.0.uptime", i, &b->uptime, uint64_t); + CTL_M2_GET("stats.arenas.0.dss", i, &b->dss, const char *); + /* + * The arena..name ctl copies the name into the caller-provided + * buffer (b->name), so we hand it a real buffer rather than reading + * into a borrowed pointer. The merged/destroyed pseudo-arenas have no + * name, and querying them would fail, so guard the read. + */ + b->has_name = (i != MALLCTL_ARENAS_ALL && i != MALLCTL_ARENAS_DESTROYED); + if (b->has_name) { + char *namep = b->name; + CTL_M1_GET("arena.0.name", i, (void *)&namep, const char *); + } +} + +static void +stats_gather_arena_decay(unsigned i, stats_arena_decay_t *d) { + CTL_M2_GET( + "stats.arenas.0.dirty_decay_ms", i, &d->dirty_decay_ms, ssize_t); + CTL_M2_GET( + "stats.arenas.0.muzzy_decay_ms", i, &d->muzzy_decay_ms, ssize_t); + CTL_M2_GET("stats.arenas.0.pactive", i, &d->pactive, size_t); + CTL_M2_GET("stats.arenas.0.pdirty", i, &d->pdirty, size_t); + CTL_M2_GET("stats.arenas.0.pmuzzy", i, &d->pmuzzy, size_t); + CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &d->dirty_npurge, uint64_t); + CTL_M2_GET( + "stats.arenas.0.dirty_nmadvise", i, &d->dirty_nmadvise, uint64_t); + CTL_M2_GET("stats.arenas.0.dirty_purged", i, &d->dirty_purged, uint64_t); + CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &d->muzzy_npurge, uint64_t); + CTL_M2_GET( + "stats.arenas.0.muzzy_nmadvise", i, &d->muzzy_nmadvise, uint64_t); + CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &d->muzzy_purged, uint64_t); +} + +static void +stats_gather_arena_alloc(unsigned i, stats_arena_alloc_t *small, + stats_arena_alloc_t *large) { + CTL_M2_GET( + "stats.arenas.0.small.allocated", i, &small->allocated, size_t); + CTL_M2_GET("stats.arenas.0.small.nmalloc", i, &small->nmalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.small.ndalloc", i, &small->ndalloc, uint64_t); + CTL_M2_GET( + "stats.arenas.0.small.nrequests", i, &small->nrequests, uint64_t); + CTL_M2_GET("stats.arenas.0.small.nfills", i, &small->nfills, uint64_t); + CTL_M2_GET("stats.arenas.0.small.nflushes", i, &small->nflushes, + uint64_t); + CTL_M2_GET( + "stats.arenas.0.large.allocated", i, &large->allocated, size_t); + CTL_M2_GET("stats.arenas.0.large.nmalloc", i, &large->nmalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.large.ndalloc", i, &large->ndalloc, uint64_t); + CTL_M2_GET( + "stats.arenas.0.large.nrequests", i, &large->nrequests, uint64_t); + CTL_M2_GET("stats.arenas.0.large.nfills", i, &large->nfills, uint64_t); + CTL_M2_GET("stats.arenas.0.large.nflushes", i, &large->nflushes, + uint64_t); +} + +/* + * nslabs + prof (which gate the table gap/skip) are read by the caller; this + * reads the remaining detail fields. mibs prepared + indexed by the caller. + */ +static void +stats_gather_arena_bin(size_t *stats_mib, size_t *arenas_bin_mib, + stats_arena_bin_t *bin) { + CTL_LEAF(arenas_bin_mib, 3, "size", &bin->reg_size, size_t); + CTL_LEAF(arenas_bin_mib, 3, "nregs", &bin->nregs, uint32_t); + CTL_LEAF(arenas_bin_mib, 3, "slab_size", &bin->slab_size, size_t); + CTL_LEAF(arenas_bin_mib, 3, "nshards", &bin->nshards, uint32_t); + CTL_LEAF(stats_mib, 5, "nmalloc", &bin->nmalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "ndalloc", &bin->ndalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "curregs", &bin->curregs, size_t); + CTL_LEAF(stats_mib, 5, "nrequests", &bin->nrequests, uint64_t); + CTL_LEAF(stats_mib, 5, "nfills", &bin->nfills, uint64_t); + CTL_LEAF(stats_mib, 5, "nflushes", &bin->nflushes, uint64_t); + CTL_LEAF(stats_mib, 5, "nreslabs", &bin->nreslabs, uint64_t); + CTL_LEAF(stats_mib, 5, "curslabs", &bin->curslabs, size_t); + CTL_LEAF(stats_mib, 5, "nonfull_slabs", &bin->nonfull_slabs, size_t); +} + +static void +stats_gather_arena_lextent(size_t *stats_mib, size_t *arenas_lextent_mib, + size_t *prof_stats_mib, unsigned j, bool prof_stats_on, + stats_arena_lextent_t *lext) { + stats_mib[4] = j; + arenas_lextent_mib[2] = j; + CTL_LEAF(stats_mib, 5, "nmalloc", &lext->nmalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "ndalloc", &lext->ndalloc, uint64_t); + CTL_LEAF(stats_mib, 5, "nrequests", &lext->nrequests, uint64_t); + CTL_LEAF(arenas_lextent_mib, 3, "size", &lext->lextent_size, size_t); + CTL_LEAF(stats_mib, 5, "curlextents", &lext->curlextents, size_t); + if (prof_stats_on) { + prof_stats_mib[3] = j; + CTL_LEAF(prof_stats_mib, 4, "live", &lext->prof_live, + prof_stats_t); + CTL_LEAF(prof_stats_mib, 4, "accum", &lext->prof_accum, + prof_stats_t); + } +} + +static void +stats_gather_arena_hpa_sec(unsigned i, stats_arena_hpa_sec_t *sec) { + CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec->sec_bytes, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec->sec_hits, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec->sec_misses, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i, + &sec->sec_dalloc_noflush, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, + &sec->sec_dalloc_flush, size_t); + CTL_M2_GET("stats.arenas.0.hpa_sec_overfills", i, &sec->sec_overfills, + size_t); +} + +static void +stats_gather_arena_pac_sec(unsigned i, stats_arena_pac_sec_t *sec) { + CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec->sec_bytes, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec->sec_hits, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec->sec_misses, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i, + &sec->sec_dalloc_noflush, size_t); + CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i, + &sec->sec_dalloc_flush, size_t); +} + +static void +stats_gather_arena_hpa_counters(unsigned i, stats_arena_hpa_counters_t *c) { + CTL_M2_GET( + "stats.arenas.0.hpa_shard.npageslabs", i, &c->npageslabs, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.nactive", i, &c->nactive, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.ndirty", i, &c->ndirty, size_t); + + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_nonhuge", i, + &c->npageslabs_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_nonhuge", i, + &c->nactive_nonhuge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_nonhuge", i, + &c->ndirty_nonhuge, size_t); + c->nretained_nonhuge = c->npageslabs_nonhuge * HUGEPAGE_PAGES + - c->nactive_nonhuge - c->ndirty_nonhuge; + + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_huge", i, + &c->npageslabs_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_huge", i, + &c->nactive_huge, size_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_huge", i, + &c->ndirty_huge, size_t); + + CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes", i, + &c->npurge_passes, uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.npurges", i, &c->npurges, uint64_t); + CTL_M2_GET( + "stats.arenas.0.hpa_shard.nhugifies", i, &c->nhugifies, uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.nhugify_failures", i, + &c->nhugify_failures, uint64_t); + CTL_M2_GET("stats.arenas.0.hpa_shard.ndehugifies", i, &c->ndehugifies, + uint64_t); +} + +/* kind is "full_slabs" or "empty_slabs". */ +static void +stats_gather_arena_hpa_slab(unsigned i, const char *kind, + stats_arena_hpa_slab_t *s) { + size_t mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(mib, 0, "stats.arenas"); + mib[2] = i; + CTL_LEAF_PREPARE(mib, 3, "hpa_shard"); + CTL_LEAF_PREPARE(mib, 4, kind); + CTL_LEAF(mib, 5, "npageslabs_huge", &s->npageslabs_huge, size_t); + CTL_LEAF(mib, 5, "nactive_huge", &s->nactive_huge, size_t); + CTL_LEAF(mib, 5, "ndirty_huge", &s->ndirty_huge, size_t); + CTL_LEAF(mib, 5, "npageslabs_nonhuge", &s->npageslabs_nonhuge, size_t); + CTL_LEAF(mib, 5, "nactive_nonhuge", &s->nactive_nonhuge, size_t); + CTL_LEAF(mib, 5, "ndirty_nonhuge", &s->ndirty_nonhuge, size_t); + s->nretained_nonhuge = s->npageslabs_nonhuge * HUGEPAGE_PAGES + - s->nactive_nonhuge - s->ndirty_nonhuge; +} + +/* mib prepared through "stats.arenas..hpa_shard.nonfull_slabs" (depth 5). */ +static void +stats_gather_arena_hpa_nonfull(size_t *mib, unsigned j, + stats_arena_hpa_slab_t *s) { + mib[5] = j; + CTL_LEAF(mib, 6, "npageslabs_huge", &s->npageslabs_huge, size_t); + CTL_LEAF(mib, 6, "nactive_huge", &s->nactive_huge, size_t); + CTL_LEAF(mib, 6, "ndirty_huge", &s->ndirty_huge, size_t); + CTL_LEAF(mib, 6, "npageslabs_nonhuge", &s->npageslabs_nonhuge, size_t); + CTL_LEAF(mib, 6, "nactive_nonhuge", &s->nactive_nonhuge, size_t); + CTL_LEAF(mib, 6, "ndirty_nonhuge", &s->ndirty_nonhuge, size_t); + s->nretained_nonhuge = s->npageslabs_nonhuge * HUGEPAGE_PAGES + - s->nactive_nonhuge - s->ndirty_nonhuge; +} + +static void +stats_gather_arena_config(stats_arena_config_t *cfg) { + CTL_GET("arenas.narenas", &cfg->narenas, unsigned); + CTL_GET("arenas.dirty_decay_ms", &cfg->dirty_decay_ms, ssize_t); + CTL_GET("arenas.muzzy_decay_ms", &cfg->muzzy_decay_ms, ssize_t); + CTL_GET("arenas.quantum", &cfg->quantum, size_t); + CTL_GET("arenas.page", &cfg->page, size_t); + CTL_GET("arenas.hugepage", &cfg->hugepage, size_t); + size_t sz = sizeof(size_t); + cfg->have_tcache_max = (je_mallctl("arenas.tcache_max", + (void *)&cfg->tcache_max, &sz, NULL, 0) == 0); + CTL_GET("arenas.nbins", &cfg->nbins, unsigned); + CTL_GET("arenas.nhbins", &cfg->nhbins, unsigned); + CTL_GET("arenas.nlextents", &cfg->nlextents, unsigned); +} + +/* mib must be prepared through "arenas.bin" (depth 2). */ +static void +stats_gather_arena_bin_meta(size_t *mib, unsigned j, + stats_arena_bin_meta_t *bm) { + mib[2] = j; + CTL_LEAF(mib, 3, "size", &bm->size, size_t); + CTL_LEAF(mib, 3, "nregs", &bm->nregs, uint32_t); + CTL_LEAF(mib, 3, "slab_size", &bm->slab_size, size_t); + CTL_LEAF(mib, 3, "nshards", &bm->nshards, uint32_t); +} + +/* mib must be prepared through "arenas.lextent" (depth 2). */ +static void +stats_gather_arena_lextent_meta(size_t *mib, unsigned j, + stats_arena_lextent_meta_t *lm) { + mib[2] = j; + CTL_LEAF(mib, 3, "size", &lm->size, size_t); +} + +static void +stats_gather_global(stats_global_t *g) { + CTL_GET("stats.allocated", &g->allocated, size_t); + CTL_GET("stats.active", &g->active, size_t); + CTL_GET("stats.metadata", &g->metadata, size_t); + CTL_GET("stats.metadata_edata", &g->metadata_edata, size_t); + CTL_GET("stats.metadata_rtree", &g->metadata_rtree, size_t); + CTL_GET("stats.metadata_thp", &g->metadata_thp, size_t); + CTL_GET("stats.resident", &g->resident, size_t); + CTL_GET("stats.mapped", &g->mapped, size_t); + CTL_GET("stats.retained", &g->retained, size_t); + CTL_GET("stats.pinned", &g->pinned, size_t); + CTL_GET("stats.zero_reallocs", &g->zero_reallocs, size_t); + + if (have_background_thread) { + CTL_GET("stats.background_thread.num_threads", + &g->num_background_threads, size_t); + CTL_GET("stats.background_thread.num_runs", + &g->background_thread_num_runs, uint64_t); + CTL_GET("stats.background_thread.run_interval", + &g->background_thread_run_interval, uint64_t); + } else { + g->num_background_threads = 0; + g->background_thread_num_runs = 0; + g->background_thread_run_interval = 0; + } +} + +/* + * Fills initialized[narenas] and *destroyed_initialized; returns the number of + * initialized arenas. + */ +static unsigned +stats_gather_arenas_initialized(unsigned narenas, bool *initialized, + bool *destroyed_initialized) { + size_t mib[3]; + size_t miblen = sizeof(mib) / sizeof(size_t); + size_t sz; + unsigned ninitialized = 0; + + xmallctlnametomib("arena.0.initialized", mib, &miblen); + for (unsigned i = 0; i < narenas; i++) { + mib[1] = i; + sz = sizeof(bool); + xmallctlbymib(mib, miblen, &initialized[i], &sz, NULL, 0); + if (initialized[i]) { + ninitialized++; + } + } + mib[1] = MALLCTL_ARENAS_DESTROYED; + sz = sizeof(bool); + xmallctlbymib(mib, miblen, destroyed_initialized, &sz, NULL, 0); + return ninitialized; +} + /******************************************************************************/ static uint64_t @@ -150,40 +579,13 @@ mutex_stats_init_cols(emitter_row_t *row, const char *table_name, col_uint64_t[mutex_counter_total_wait_time_ps].width = 10; } +/* + * Read one mutex's counters (leaf "name" under the prepared mib) into the + * emitter columns. Shared by the global and per-arena mutex tables, whose rows + * differ only in the mib and mutex name they pass. + */ static void -mutex_stats_read_global(size_t mib[], size_t miblen, const char *name, - emitter_col_t *col_name, - emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], - emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], - uint64_t uptime) { - CTL_LEAF_PREPARE(mib, miblen, name); - size_t miblen_name = miblen + 1; - - col_name->str_val = name; - - emitter_col_t *dst; -#define EMITTER_TYPE_uint32_t emitter_type_uint32 -#define EMITTER_TYPE_uint64_t emitter_type_uint64 -#define OP(counter, counter_type, human, derived, base_counter) \ - dst = &col_##counter_type[mutex_counter_##counter]; \ - dst->type = EMITTER_TYPE_##counter_type; \ - if (!derived) { \ - CTL_LEAF(mib, miblen_name, #counter, \ - (counter_type *)&dst->bool_val, counter_type); \ - } else { \ - emitter_col_t *base = \ - &col_##counter_type[mutex_counter_##base_counter]; \ - dst->counter_type##_val = (counter_type)rate_per_second( \ - base->counter_type##_val, uptime); \ - } - MUTEX_PROF_COUNTERS -#undef OP -#undef EMITTER_TYPE_uint32_t -#undef EMITTER_TYPE_uint64_t -} - -static void -mutex_stats_read_arena(size_t mib[], size_t miblen, const char *name, +mutex_stats_read(size_t mib[], size_t miblen, const char *name, emitter_col_t *col_name, emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], @@ -273,42 +675,40 @@ mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, #undef EMITTER_TYPE_uint64_t } -#define COL_DECLARE(column_name) emitter_col_t col_##column_name; - -#define COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ - emitter_col_init(&col_##column_name, &row_name); \ - col_##column_name.justify = emitter_justify_##left_or_right; \ - col_##column_name.width = col_width; \ - col_##column_name.type = emitter_type_##etype; - -#define COL(row_name, column_name, left_or_right, col_width, etype) \ - COL_DECLARE(column_name); \ - COL_INIT(row_name, column_name, left_or_right, col_width, etype) - -#define COL_HDR_DECLARE(column_name) \ - COL_DECLARE(column_name); \ - emitter_col_t header_##column_name; - -#define COL_HDR_INIT( \ - row_name, column_name, human, left_or_right, col_width, etype) \ - COL_INIT(row_name, column_name, left_or_right, col_width, etype) \ - emitter_col_init(&header_##column_name, &header_##row_name); \ - header_##column_name.justify = emitter_justify_##left_or_right; \ - header_##column_name.width = col_width; \ - header_##column_name.type = emitter_type_title; \ - header_##column_name.str_val = human ? human : #column_name; - -#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ - COL_HDR_DECLARE(column_name) \ - COL_HDR_INIT( \ - row_name, column_name, human, left_or_right, col_width, etype) +/* + * Set a per-size-class row's "size" column. Above the slow-growth threshold, + * display the bucket bounds as "(prev_size,size]". This notation does not + * imply that every integer in the interval is a valid usable size. Smaller + * classes are displayed as a single size. buf must outlive row emission. + * (This is table labeling only; the ordered JSON arrays imply the bounds.) + */ +static void +stats_size_col_set(emitter_col_t *col, size_t size, size_t prev_size, char *buf, + size_t bufsz) { + if (size > USIZE_GROW_SLOW_THRESHOLD) { + malloc_snprintf(buf, bufsz, "(%zu,%zu]", prev_size, size); + col->type = emitter_type_title; + col->str_val = buf; + } else { + col->type = emitter_type_size; + col->size_val = size; + } +} JEMALLOC_COLD static void stats_arena_bins_print( emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { + /* + * Streaming table (deviates from the gather/emit split): the SC_NBINS + * rows are gathered and emitted one at a time (stats_gather_arena_bin + * per row + inline emit, with all-zero runs collapsed to "---"). This + * preserves O(1) memory. Full gather-all-rows-before-emit separation + * would require buffering the table; a separate per-row emit helper + * would not. The lextents / extents / hpa-nonfull tables follow the + * same pattern. + */ size_t page; - bool in_gap, in_gap_prev; unsigned nbins, j; CTL_GET("arenas.page", &page, size_t); @@ -399,61 +799,44 @@ stats_arena_bins_print( CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.bins"); } - for (j = 0, in_gap = false; j < nbins; j++) { - uint64_t nslabs; - size_t reg_size, slab_size, curregs; - size_t curslabs; - size_t nonfull_slabs; - uint32_t nregs, nshards; - uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; - uint64_t nreslabs; - prof_stats_t prof_live; - prof_stats_t prof_accum; - + emitter_table_sparse_begin(emitter); + for (j = 0; j < nbins; j++) { + stats_arena_bin_t bin; stats_arenas_mib[4] = j; arenas_bin_mib[2] = j; - CTL_LEAF(stats_arenas_mib, 5, "nslabs", &nslabs, uint64_t); + CTL_LEAF(stats_arenas_mib, 5, "nslabs", &bin.nslabs, uint64_t); if (prof_stats_on) { prof_stats_mib[3] = j; - CTL_LEAF(prof_stats_mib, 4, "live", &prof_live, + CTL_LEAF(prof_stats_mib, 4, "live", &bin.prof_live, prof_stats_t); - CTL_LEAF(prof_stats_mib, 4, "accum", &prof_accum, + CTL_LEAF(prof_stats_mib, 4, "accum", &bin.prof_accum, prof_stats_t); } - in_gap_prev = in_gap; + bool is_gap; if (prof_stats_on) { - in_gap = (nslabs == 0 && prof_accum.count == 0); + is_gap = (bin.nslabs == 0 && bin.prof_accum.count == 0); } else { - in_gap = (nslabs == 0); + is_gap = (bin.nslabs == 0); } - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } - - if (in_gap && !emitter_outputs_json(emitter)) { + if (is_gap && !emitter_outputs_json(emitter)) { + emitter_table_sparse_row(emitter, &row, true); continue; } - CTL_LEAF(arenas_bin_mib, 3, "size", ®_size, size_t); - CTL_LEAF(arenas_bin_mib, 3, "nregs", &nregs, uint32_t); - CTL_LEAF(arenas_bin_mib, 3, "slab_size", &slab_size, size_t); - CTL_LEAF(arenas_bin_mib, 3, "nshards", &nshards, uint32_t); - CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "curregs", &curregs, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "nrequests", &nrequests, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "nfills", &nfills, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "nflushes", &nflushes, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "nreslabs", &nreslabs, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "curslabs", &curslabs, size_t); - CTL_LEAF(stats_arenas_mib, 5, "nonfull_slabs", &nonfull_slabs, - size_t); + /* + * Per-size-class tables stream: gather one row, emit it, reuse + * buffers. Full gather-all-rows-before-emit separation would + * require table-sized storage, so gathering and emission remain + * interleaved across rows. In particular the per-bin mutex + * counters are read here (the mutex subsystem reads straight into + * emitter columns) rather than up front. A separate per-row emit + * helper could further decouple the code without buffering. + */ + stats_gather_arena_bin(stats_arenas_mib, arenas_bin_mib, &bin); if (mutex) { mutex_stats_read_arena_bin(stats_arenas_mib, 5, @@ -462,33 +845,33 @@ stats_arena_bins_print( emitter_json_object_begin(emitter); emitter_json_kv( - emitter, "nmalloc", emitter_type_uint64, &nmalloc); + emitter, "nmalloc", emitter_type_uint64, &bin.nmalloc); emitter_json_kv( - emitter, "ndalloc", emitter_type_uint64, &ndalloc); + emitter, "ndalloc", emitter_type_uint64, &bin.ndalloc); emitter_json_kv( - emitter, "curregs", emitter_type_size, &curregs); + emitter, "curregs", emitter_type_size, &bin.curregs); emitter_json_kv( - emitter, "nrequests", emitter_type_uint64, &nrequests); + emitter, "nrequests", emitter_type_uint64, &bin.nrequests); if (prof_stats_on) { emitter_json_kv(emitter, "prof_live_requested", - emitter_type_uint64, &prof_live.req_sum); + emitter_type_uint64, &bin.prof_live.req_sum); emitter_json_kv(emitter, "prof_live_count", - emitter_type_uint64, &prof_live.count); + emitter_type_uint64, &bin.prof_live.count); emitter_json_kv(emitter, "prof_accum_requested", - emitter_type_uint64, &prof_accum.req_sum); + emitter_type_uint64, &bin.prof_accum.req_sum); emitter_json_kv(emitter, "prof_accum_count", - emitter_type_uint64, &prof_accum.count); + emitter_type_uint64, &bin.prof_accum.count); } emitter_json_kv( - emitter, "nfills", emitter_type_uint64, &nfills); + emitter, "nfills", emitter_type_uint64, &bin.nfills); emitter_json_kv( - emitter, "nflushes", emitter_type_uint64, &nflushes); + emitter, "nflushes", emitter_type_uint64, &bin.nflushes); emitter_json_kv( - emitter, "nreslabs", emitter_type_uint64, &nreslabs); + emitter, "nreslabs", emitter_type_uint64, &bin.nreslabs); emitter_json_kv( - emitter, "curslabs", emitter_type_size, &curslabs); + emitter, "curslabs", emitter_type_size, &bin.curslabs); emitter_json_kv(emitter, "nonfull_slabs", emitter_type_size, - &nonfull_slabs); + &bin.nonfull_slabs); if (mutex) { emitter_json_object_kv_begin(emitter, "mutex"); mutex_stats_emit( @@ -497,13 +880,13 @@ stats_arena_bins_print( } emitter_json_object_end(emitter); - size_t availregs = nregs * curslabs; + size_t availregs = bin.nregs * bin.curslabs; char util[6]; if (get_rate_str( - (uint64_t)curregs, (uint64_t)availregs, util)) { + (uint64_t)bin.curregs, (uint64_t)availregs, util)) { if (availregs == 0) { malloc_snprintf(util, sizeof(util), "1"); - } else if (curregs > availregs) { + } else if (bin.curregs > availregs) { /* * Race detected: the counters were read in * separate mallctl calls and concurrent @@ -516,57 +899,55 @@ stats_arena_bins_print( } } - col_size.size_val = reg_size; + col_size.size_val = bin.reg_size; col_ind.unsigned_val = j; - col_allocated.size_val = curregs * reg_size; - col_nmalloc.uint64_val = nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); - col_ndalloc.uint64_val = ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); - col_nrequests.uint64_val = nrequests; + col_allocated.size_val = bin.curregs * bin.reg_size; + col_nmalloc.uint64_val = bin.nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(bin.nmalloc, uptime); + col_ndalloc.uint64_val = bin.ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(bin.ndalloc, uptime); + col_nrequests.uint64_val = bin.nrequests; col_nrequests_ps.uint64_val = rate_per_second( - nrequests, uptime); + bin.nrequests, uptime); if (prof_stats_on) { - col_prof_live_requested.uint64_val = prof_live.req_sum; - col_prof_live_count.uint64_val = prof_live.count; + col_prof_live_requested.uint64_val = bin.prof_live.req_sum; + col_prof_live_count.uint64_val = bin.prof_live.count; col_prof_accum_requested.uint64_val = - prof_accum.req_sum; - col_prof_accum_count.uint64_val = prof_accum.count; + bin.prof_accum.req_sum; + col_prof_accum_count.uint64_val = bin.prof_accum.count; } - col_nshards.unsigned_val = nshards; - col_curregs.size_val = curregs; - col_curslabs.size_val = curslabs; - col_nonfull_slabs.size_val = nonfull_slabs; - col_regs.unsigned_val = nregs; - col_pgs.size_val = slab_size / page; + col_nshards.unsigned_val = bin.nshards; + col_curregs.size_val = bin.curregs; + col_curslabs.size_val = bin.curslabs; + col_nonfull_slabs.size_val = bin.nonfull_slabs; + col_regs.unsigned_val = bin.nregs; + col_pgs.size_val = bin.slab_size / page; col_util.str_val = util; - col_nfills.uint64_val = nfills; - col_nfills_ps.uint64_val = rate_per_second(nfills, uptime); - col_nflushes.uint64_val = nflushes; - col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime); - col_nslabs.uint64_val = nslabs; - col_nreslabs.uint64_val = nreslabs; - col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime); + col_nfills.uint64_val = bin.nfills; + col_nfills_ps.uint64_val = rate_per_second(bin.nfills, uptime); + col_nflushes.uint64_val = bin.nflushes; + col_nflushes_ps.uint64_val = rate_per_second(bin.nflushes, uptime); + col_nslabs.uint64_val = bin.nslabs; + col_nreslabs.uint64_val = bin.nreslabs; + col_nreslabs_ps.uint64_val = rate_per_second(bin.nreslabs, uptime); /* * Note that mutex columns were initialized above, if mutex == * true. */ - emitter_table_row(emitter, &row); + emitter_table_sparse_row(emitter, &row, is_gap); } emitter_json_array_end(emitter); /* Close "bins". */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); } JEMALLOC_COLD static void stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { + /* Streaming table; see stats_arena_bins_print for the rationale. */ unsigned nbins, nlextents, j; - bool in_gap, in_gap_prev; CTL_GET("arenas.nbins", &nbins, unsigned); CTL_GET("arenas.nlextents", &nlextents, unsigned); @@ -597,8 +978,13 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { COL_HDR_INIT(row, prof_live_count, NULL, right, 17, uint64) COL_HDR_INIT(row, prof_accum_requested, NULL, right, 21, uint64) COL_HDR_INIT(row, prof_accum_count, NULL, right, 17, uint64) + col_prof_live_requested.json_key = "prof_live_requested"; + col_prof_live_count.json_key = "prof_live_count"; + col_prof_accum_requested.json_key = "prof_accum_requested"; + col_prof_accum_count.json_key = "prof_accum_count"; } COL_HDR(row, curlextents, NULL, right, 13, size) + col_curlextents.json_key = "curlextents"; /* As with bins, we label the large extents table. */ header_size.width -= 6; @@ -619,89 +1005,58 @@ stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.lextents"); } - for (j = 0, in_gap = false; j < nlextents; j++) { - uint64_t nmalloc, ndalloc, nrequests; - size_t lextent_size, curlextents; - prof_stats_t prof_live; - prof_stats_t prof_accum; + /* + * Carry the previous size class's size forward for the "(prev,size]" + * range label instead of looking it up by a large size-class index: + * with large size classes disabled (the default) that index may be out + * of range, and indexing sz_index2size_tab there would be unsafe. The + * class before the first large one is the last bin; sz_index2size_compute + * derives it arithmetically (no table access). + */ + size_t prev_size = sz_index2size_compute(nbins - 1); + emitter_table_sparse_begin(emitter); + for (j = 0; j < nlextents; j++) { + stats_arena_lextent_t lext; + stats_gather_arena_lextent(stats_arenas_mib, arenas_lextent_mib, + prof_stats_mib, j, prof_stats_on, &lext); - stats_arenas_mib[4] = j; - arenas_lextent_mib[2] = j; - - CTL_LEAF(stats_arenas_mib, 5, "nmalloc", &nmalloc, uint64_t); - CTL_LEAF(stats_arenas_mib, 5, "ndalloc", &ndalloc, uint64_t); - CTL_LEAF( - stats_arenas_mib, 5, "nrequests", &nrequests, uint64_t); - - in_gap_prev = in_gap; - in_gap = (nrequests == 0); - - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } - - CTL_LEAF(arenas_lextent_mib, 3, "size", &lextent_size, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "curlextents", &curlextents, size_t); + bool is_gap = (lext.nrequests == 0); + char size_buf[48]; + stats_size_col_set(&col_size, lext.lextent_size, prev_size, + size_buf, sizeof(size_buf)); + prev_size = lext.lextent_size; + col_ind.unsigned_val = nbins + j; + col_allocated.size_val = lext.curlextents * lext.lextent_size; + col_nmalloc.uint64_val = lext.nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(lext.nmalloc, uptime); + col_ndalloc.uint64_val = lext.ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(lext.ndalloc, uptime); + col_nrequests.uint64_val = lext.nrequests; + col_nrequests_ps.uint64_val = rate_per_second( + lext.nrequests, uptime); if (prof_stats_on) { - prof_stats_mib[3] = j; - CTL_LEAF(prof_stats_mib, 4, "live", &prof_live, - prof_stats_t); - CTL_LEAF(prof_stats_mib, 4, "accum", &prof_accum, - prof_stats_t); + col_prof_live_requested.uint64_val = lext.prof_live.req_sum; + col_prof_live_count.uint64_val = lext.prof_live.count; + col_prof_accum_requested.uint64_val = + lext.prof_accum.req_sum; + col_prof_accum_count.uint64_val = lext.prof_accum.count; } + col_curlextents.size_val = lext.curlextents; emitter_json_object_begin(emitter); - if (prof_stats_on) { - emitter_json_kv(emitter, "prof_live_requested", - emitter_type_uint64, &prof_live.req_sum); - emitter_json_kv(emitter, "prof_live_count", - emitter_type_uint64, &prof_live.count); - emitter_json_kv(emitter, "prof_accum_requested", - emitter_type_uint64, &prof_accum.req_sum); - emitter_json_kv(emitter, "prof_accum_count", - emitter_type_uint64, &prof_accum.count); - } - emitter_json_kv( - emitter, "curlextents", emitter_type_size, &curlextents); + emitter_sparse_row(emitter, &row, is_gap); emitter_json_object_end(emitter); - - col_size.size_val = lextent_size; - col_ind.unsigned_val = nbins + j; - col_allocated.size_val = curlextents * lextent_size; - col_nmalloc.uint64_val = nmalloc; - col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); - col_ndalloc.uint64_val = ndalloc; - col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); - col_nrequests.uint64_val = nrequests; - col_nrequests_ps.uint64_val = rate_per_second( - nrequests, uptime); - if (prof_stats_on) { - col_prof_live_requested.uint64_val = prof_live.req_sum; - col_prof_live_count.uint64_val = prof_live.count; - col_prof_accum_requested.uint64_val = - prof_accum.req_sum; - col_prof_accum_count.uint64_val = prof_accum.count; - } - col_curlextents.size_val = curlextents; - - if (!in_gap) { - emitter_table_row(emitter, &row); - } } emitter_json_array_end(emitter); /* Close "lextents". */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); } JEMALLOC_COLD static void stats_arena_extents_print(emitter_t *emitter, unsigned i) { + /* Streaming table; see stats_arena_bins_print for the rationale. */ unsigned j; - bool in_gap, in_gap_prev; emitter_row_t header_row; emitter_row_init(&header_row); emitter_row_t row; @@ -731,253 +1086,163 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) { stats_arenas_mib[2] = i; CTL_LEAF_PREPARE(stats_arenas_mib, 3, "extents"); - in_gap = false; + emitter_table_sparse_begin(emitter); for (j = 0; j < SC_NPSIZES; j++) { - size_t ndirty, nmuzzy, nretained, npinned, total, - dirty_bytes, muzzy_bytes, retained_bytes, pinned_bytes, - total_bytes; - stats_arenas_mib[4] = j; + stats_arena_extent_t e; + stats_gather_arena_extent(stats_arenas_mib, j, &e); - CTL_LEAF(stats_arenas_mib, 5, "ndirty", &ndirty, size_t); - CTL_LEAF(stats_arenas_mib, 5, "nmuzzy", &nmuzzy, size_t); - CTL_LEAF(stats_arenas_mib, 5, "nretained", &nretained, size_t); - CTL_LEAF(stats_arenas_mib, 5, "npinned", &npinned, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "dirty_bytes", &dirty_bytes, size_t); - CTL_LEAF( - stats_arenas_mib, 5, "muzzy_bytes", &muzzy_bytes, size_t); - CTL_LEAF(stats_arenas_mib, 5, "retained_bytes", &retained_bytes, - size_t); - CTL_LEAF(stats_arenas_mib, 5, "pinned_bytes", &pinned_bytes, - size_t); - - total = ndirty + nmuzzy + nretained + npinned; - total_bytes = dirty_bytes + muzzy_bytes + retained_bytes - + pinned_bytes; - - in_gap_prev = in_gap; - in_gap = (total == 0); - - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } + bool is_gap = (e.ntotal == 0); emitter_json_object_begin(emitter); - emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); - emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy); + emitter_json_kv(emitter, "ndirty", emitter_type_size, &e.ndirty); + emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &e.nmuzzy); emitter_json_kv( - emitter, "nretained", emitter_type_size, &nretained); + emitter, "nretained", emitter_type_size, &e.nretained); emitter_json_kv( - emitter, "npinned", emitter_type_size, &npinned); + emitter, "npinned", emitter_type_size, &e.npinned); emitter_json_kv( - emitter, "dirty_bytes", emitter_type_size, &dirty_bytes); + emitter, "dirty_bytes", emitter_type_size, &e.dirty_bytes); emitter_json_kv( - emitter, "muzzy_bytes", emitter_type_size, &muzzy_bytes); + emitter, "muzzy_bytes", emitter_type_size, &e.muzzy_bytes); emitter_json_kv(emitter, "retained_bytes", emitter_type_size, - &retained_bytes); + &e.retained_bytes); emitter_json_kv(emitter, "pinned_bytes", emitter_type_size, - &pinned_bytes); + &e.pinned_bytes); emitter_json_object_end(emitter); - col_size.size_val = sz_pind2sz(j); - col_ind.size_val = j; - col_ndirty.size_val = ndirty; - col_dirty.size_val = dirty_bytes; - col_nmuzzy.size_val = nmuzzy; - col_muzzy.size_val = muzzy_bytes; - col_nretained.size_val = nretained; - col_retained.size_val = retained_bytes; - col_npinned.size_val = npinned; - col_pinned.size_val = pinned_bytes; - col_ntotal.size_val = total; - col_total.size_val = total_bytes; + char size_buf[48]; + stats_size_col_set(&col_size, sz_pind2sz(j), + j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); + col_ind.unsigned_val = j; + col_ndirty.size_val = e.ndirty; + col_dirty.size_val = e.dirty_bytes; + col_nmuzzy.size_val = e.nmuzzy; + col_muzzy.size_val = e.muzzy_bytes; + col_nretained.size_val = e.nretained; + col_retained.size_val = e.retained_bytes; + col_npinned.size_val = e.npinned; + col_pinned.size_val = e.pinned_bytes; + col_ntotal.size_val = e.ntotal; + col_total.size_val = e.total_bytes; - if (!in_gap) { - emitter_table_row(emitter, &row); - } + emitter_table_sparse_row(emitter, &row, is_gap); } emitter_json_array_end(emitter); /* Close "extents". */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); +} + +static void +stats_emit_arena_hpa_sec(emitter_t *emitter, const stats_arena_hpa_sec_t *sec) { + emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache", + emitter_type_size, &sec->sec_bytes); + emitter_kv(emitter, "sec_hits", "Total hits in small extent cache", + emitter_type_size, &sec->sec_hits); + emitter_kv(emitter, "sec_misses", "Total misses in small extent cache", + emitter_type_size, &sec->sec_misses); + emitter_kv(emitter, "sec_dalloc_noflush", + "Dalloc calls without flush in small extent cache", + emitter_type_size, &sec->sec_dalloc_noflush); + emitter_kv(emitter, "sec_dalloc_flush", + "Dalloc calls with flush in small extent cache", emitter_type_size, + &sec->sec_dalloc_flush); + emitter_kv(emitter, "sec_overfills", + "sec_fill calls that went over max_bytes", emitter_type_size, + &sec->sec_overfills); } static void stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) { - size_t sec_bytes; - size_t sec_hits; - size_t sec_misses; - size_t sec_dalloc_flush; - size_t sec_dalloc_noflush; - size_t sec_overfills; - CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t); - emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache", - emitter_type_size, &sec_bytes); - CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec_hits, size_t); - emitter_kv(emitter, "sec_hits", "Total hits in small extent cache", - emitter_type_size, &sec_hits); - CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec_misses, size_t); - emitter_kv(emitter, "sec_misses", "Total misses in small extent cache", - emitter_type_size, &sec_misses); - CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i, - &sec_dalloc_noflush, size_t); - emitter_kv(emitter, "sec_dalloc_noflush", - "Dalloc calls without flush in small extent cache", - emitter_type_size, &sec_dalloc_noflush); - CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, &sec_dalloc_flush, - size_t); - emitter_kv(emitter, "sec_dalloc_flush", - "Dalloc calls with flush in small extent cache", emitter_type_size, - &sec_dalloc_flush); - CTL_M2_GET( - "stats.arenas.0.hpa_sec_overfills", i, &sec_overfills, size_t); - emitter_kv(emitter, "sec_overfills", - "sec_fill calls that went over max_bytes", emitter_type_size, - &sec_overfills); + stats_arena_hpa_sec_t sec; + stats_gather_arena_hpa_sec(i, &sec); + stats_emit_arena_hpa_sec(emitter, &sec); +} + +static void +stats_emit_arena_pac_sec(emitter_t *emitter, const stats_arena_pac_sec_t *sec) { + emitter_kv(emitter, "pac_sec_bytes", "Bytes in PAC small extent cache", + emitter_type_size, &sec->sec_bytes); + emitter_kv(emitter, "pac_sec_hits", "Total hits in PAC small extent cache", + emitter_type_size, &sec->sec_hits); + emitter_kv(emitter, "pac_sec_misses", + "Total misses in PAC small extent cache", emitter_type_size, + &sec->sec_misses); + emitter_kv(emitter, "pac_sec_dalloc_noflush", + "Dalloc calls without flush in PAC small extent cache", + emitter_type_size, &sec->sec_dalloc_noflush); + emitter_kv(emitter, "pac_sec_dalloc_flush", + "Dalloc calls with flush in PAC small extent cache", + emitter_type_size, &sec->sec_dalloc_flush); } static void stats_arena_pac_sec_print(emitter_t *emitter, unsigned i) { - size_t sec_bytes; - size_t sec_hits; - size_t sec_misses; - size_t sec_dalloc_flush; - size_t sec_dalloc_noflush; - CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec_bytes, size_t); - emitter_kv(emitter, "pac_sec_bytes", - "Bytes in PAC small extent cache", - emitter_type_size, &sec_bytes); - CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec_hits, size_t); - emitter_kv(emitter, "pac_sec_hits", - "Total hits in PAC small extent cache", - emitter_type_size, &sec_hits); - CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec_misses, size_t); - emitter_kv(emitter, "pac_sec_misses", - "Total misses in PAC small extent cache", - emitter_type_size, &sec_misses); - CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i, - &sec_dalloc_noflush, size_t); - emitter_kv(emitter, "pac_sec_dalloc_noflush", - "Dalloc calls without flush in PAC small extent cache", - emitter_type_size, &sec_dalloc_noflush); - CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i, &sec_dalloc_flush, - size_t); - emitter_kv(emitter, "pac_sec_dalloc_flush", - "Dalloc calls with flush in PAC small extent cache", - emitter_type_size, &sec_dalloc_flush); + stats_arena_pac_sec_t sec; + stats_gather_arena_pac_sec(i, &sec); + stats_emit_arena_pac_sec(emitter, &sec); +} + +static void +stats_emit_arena_hpa_counters(emitter_t *emitter, + const stats_arena_hpa_counters_t *c, uint64_t uptime) { + /* Merged pageslab / page counts (broken down by huginess below). */ + emitter_kv(emitter, "npageslabs", "npageslabs", emitter_type_size, + &c->npageslabs); + emitter_kv(emitter, "nactive", "nactive", emitter_type_size, + &c->nactive); + emitter_kv(emitter, "ndirty", "ndirty", emitter_type_size, &c->ndirty); + + uint64_t npurge_passes_ps = rate_per_second(c->npurge_passes, uptime); + uint64_t npurges_ps = rate_per_second(c->npurges, uptime); + uint64_t nhugifies_ps = rate_per_second(c->nhugifies, uptime); + uint64_t nhugify_failures_ps = + rate_per_second(c->nhugify_failures, uptime); + uint64_t ndehugifies_ps = rate_per_second(c->ndehugifies, uptime); + emitter_kv_note(emitter, "npurge_passes", "npurge_passes", + emitter_type_uint64, &c->npurge_passes, "per_sec", + emitter_type_uint64, &npurge_passes_ps); + emitter_kv_note(emitter, "npurges", "npurges", emitter_type_uint64, + &c->npurges, "per_sec", emitter_type_uint64, &npurges_ps); + emitter_kv_note(emitter, "nhugifies", "nhugifies", emitter_type_uint64, + &c->nhugifies, "per_sec", emitter_type_uint64, &nhugifies_ps); + emitter_kv_note(emitter, "nhugify_failures", "nhugify_failures", + emitter_type_uint64, &c->nhugify_failures, "per_sec", + emitter_type_uint64, &nhugify_failures_ps); + emitter_kv_note(emitter, "ndehugifies", "ndehugifies", + emitter_type_uint64, &c->ndehugifies, "per_sec", + emitter_type_uint64, &ndehugifies_ps); + + emitter_dict_begin(emitter, "slabs", "slabs"); + emitter_kv(emitter, "npageslabs_nonhuge", "npageslabs_nonhuge", + emitter_type_size, &c->npageslabs_nonhuge); + emitter_kv(emitter, "nactive_nonhuge", "nactive_nonhuge", + emitter_type_size, &c->nactive_nonhuge); + emitter_kv(emitter, "ndirty_nonhuge", "ndirty_nonhuge", + emitter_type_size, &c->ndirty_nonhuge); + emitter_kv(emitter, "nretained_nonhuge", "nretained_nonhuge", + emitter_type_size, &c->nretained_nonhuge); + emitter_kv(emitter, "npageslabs_huge", "npageslabs_huge", + emitter_type_size, &c->npageslabs_huge); + emitter_kv(emitter, "nactive_huge", "nactive_huge", emitter_type_size, + &c->nactive_huge); + emitter_kv(emitter, "ndirty_huge", "ndirty_huge", emitter_type_size, + &c->ndirty_huge); + emitter_dict_end(emitter); } static void stats_arena_hpa_shard_counters_print( emitter_t *emitter, unsigned i, uint64_t uptime) { - size_t npageslabs; - size_t nactive; - size_t ndirty; + stats_arena_hpa_counters_t c; + stats_gather_arena_hpa_counters(i, &c); + stats_emit_arena_hpa_counters(emitter, &c, uptime); - size_t npageslabs_nonhuge; - size_t nactive_nonhuge; - size_t ndirty_nonhuge; - size_t nretained_nonhuge; - - size_t npageslabs_huge; - size_t nactive_huge; - size_t ndirty_huge; - - uint64_t npurge_passes; - uint64_t npurges; - uint64_t nhugifies; - uint64_t nhugify_failures; - uint64_t ndehugifies; - - CTL_M2_GET( - "stats.arenas.0.hpa_shard.npageslabs", i, &npageslabs, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.nactive", i, &nactive, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.ndirty", i, &ndirty, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_nonhuge", i, - &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_nonhuge", i, - &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_nonhuge", i, - &ndirty_nonhuge, size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_huge", i, - &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_huge", i, - &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_huge", i, - &ndirty_huge, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes", i, &npurge_passes, - uint64_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.npurges", i, &npurges, uint64_t); - CTL_M2_GET( - "stats.arenas.0.hpa_shard.nhugifies", i, &nhugifies, uint64_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.nhugify_failures", i, - &nhugify_failures, uint64_t); - CTL_M2_GET( - "stats.arenas.0.hpa_shard.ndehugifies", i, &ndehugifies, uint64_t); - - emitter_table_printf(emitter, - "HPA shard stats:\n" - " Pageslabs: %zu (%zu huge, %zu nonhuge)\n" - " Active pages: %zu (%zu huge, %zu nonhuge)\n" - " Dirty pages: %zu (%zu huge, %zu nonhuge)\n" - " Retained pages: %zu\n" - " Purge passes: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Purges: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Hugeifies: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Hugify failures: %" FMTu64 " (%" FMTu64 - " / sec)\n" - " Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n", - npageslabs, npageslabs_huge, npageslabs_nonhuge, nactive, - nactive_huge, nactive_nonhuge, ndirty, ndirty_huge, ndirty_nonhuge, - nretained_nonhuge, npurge_passes, - rate_per_second(npurge_passes, uptime), npurges, - rate_per_second(npurges, uptime), nhugifies, - rate_per_second(nhugifies, uptime), nhugify_failures, - rate_per_second(nhugify_failures, uptime), ndehugifies, - rate_per_second(ndehugifies, uptime)); - - emitter_json_kv(emitter, "npageslabs", emitter_type_size, &npageslabs); - emitter_json_kv(emitter, "nactive", emitter_type_size, &nactive); - emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); - - emitter_json_kv( - emitter, "npurge_passes", emitter_type_uint64, &npurge_passes); - emitter_json_kv(emitter, "npurges", emitter_type_uint64, &npurges); - emitter_json_kv(emitter, "nhugifies", emitter_type_uint64, &nhugifies); - emitter_json_kv(emitter, "nhugify_failures", emitter_type_uint64, - &nhugify_failures); - emitter_json_kv( - emitter, "ndehugifies", emitter_type_uint64, &ndehugifies); - - emitter_json_object_kv_begin(emitter, "slabs"); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &npageslabs_nonhuge); - emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); - emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); - - emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); - emitter_json_object_end(emitter); /* End "slabs" */ - - /* alloc_batch stats */ + /* + * The extent-allocation distribution below is a small fixed-size + * (SEC_MAX_NALLOCS + 1) table with its own leaf ctls, so it is + * gathered into stack arrays and emitted (table then JSON) inline + * rather than through a stats_gather/stats_emit pair. + */ uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1]; uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1]; uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1]; @@ -1054,6 +1319,43 @@ stats_arena_hpa_shard_counters_print( emitter_json_array_end(emitter); /* End "alloc_batch" */ } +static void +stats_emit_arena_hpa_slab_json(emitter_t *emitter, + const stats_arena_hpa_slab_t *s, const char *json_key) { + emitter_json_object_kv_begin(emitter, json_key); + emitter_json_kv( + emitter, "npageslabs_huge", emitter_type_size, &s->npageslabs_huge); + emitter_json_kv( + emitter, "nactive_huge", emitter_type_size, &s->nactive_huge); + emitter_json_kv( + emitter, "ndirty_huge", emitter_type_size, &s->ndirty_huge); + emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, + &s->npageslabs_nonhuge); + emitter_json_kv( + emitter, "nactive_nonhuge", emitter_type_size, &s->nactive_nonhuge); + emitter_json_kv( + emitter, "ndirty_nonhuge", emitter_type_size, &s->ndirty_nonhuge); + emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, + &s->nretained_nonhuge); + emitter_json_object_end(emitter); +} + +/* Fill the shared count columns of one pageslab row from s. */ +static void +stats_hpa_slab_cols_set(const stats_arena_hpa_slab_t *s, + emitter_col_t *npageslabs_huge, emitter_col_t *nactive_huge, + emitter_col_t *ndirty_huge, emitter_col_t *npageslabs_nonhuge, + emitter_col_t *nactive_nonhuge, emitter_col_t *ndirty_nonhuge, + emitter_col_t *nretained_nonhuge) { + npageslabs_huge->size_val = s->npageslabs_huge; + nactive_huge->size_val = s->nactive_huge; + ndirty_huge->size_val = s->ndirty_huge; + npageslabs_nonhuge->size_val = s->npageslabs_nonhuge; + nactive_nonhuge->size_val = s->nactive_nonhuge; + ndirty_nonhuge->size_val = s->ndirty_nonhuge; + nretained_nonhuge->size_val = s->nretained_nonhuge; +} + static void stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_row_t header_row; @@ -1061,102 +1363,6 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { emitter_row_t row; emitter_row_init(&row); - size_t npageslabs_huge; - size_t nactive_huge; - size_t ndirty_huge; - - size_t npageslabs_nonhuge; - size_t nactive_nonhuge; - size_t ndirty_nonhuge; - size_t nretained_nonhuge; - - /* Full slab stats. */ - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_huge", i, - &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_huge", i, - &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_huge", i, - &ndirty_huge, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.npageslabs_nonhuge", i, - &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.nactive_nonhuge", i, - &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.full_slabs.ndirty_nonhuge", i, - &ndirty_nonhuge, size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - emitter_table_printf(emitter, - " In full slabs:\n" - " npageslabs: %zu huge, %zu nonhuge\n" - " nactive: %zu huge, %zu nonhuge \n" - " ndirty: %zu huge, %zu nonhuge \n" - " nretained: 0 huge, %zu nonhuge \n", - npageslabs_huge, npageslabs_nonhuge, nactive_huge, nactive_nonhuge, - ndirty_huge, ndirty_nonhuge, nretained_nonhuge); - - emitter_json_object_kv_begin(emitter, "full_slabs"); - emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &npageslabs_nonhuge); - emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); - emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); - emitter_json_object_end(emitter); /* End "full_slabs" */ - - /* Next, empty slab stats. */ - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_huge", i, - &npageslabs_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_huge", i, - &nactive_huge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_huge", i, - &ndirty_huge, size_t); - - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.npageslabs_nonhuge", i, - &npageslabs_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.nactive_nonhuge", i, - &nactive_nonhuge, size_t); - CTL_M2_GET("stats.arenas.0.hpa_shard.empty_slabs.ndirty_nonhuge", i, - &ndirty_nonhuge, size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - emitter_table_printf(emitter, - " In empty slabs:\n" - " npageslabs: %zu huge, %zu nonhuge\n" - " nactive: %zu huge, %zu nonhuge \n" - " ndirty: %zu huge, %zu nonhuge \n" - " nretained: 0 huge, %zu nonhuge \n", - npageslabs_huge, npageslabs_nonhuge, nactive_huge, nactive_nonhuge, - ndirty_huge, ndirty_nonhuge, nretained_nonhuge); - - emitter_json_object_kv_begin(emitter, "empty_slabs"); - emitter_json_kv( - emitter, "npageslabs_huge", emitter_type_size, &npageslabs_huge); - emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); - emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); - emitter_json_kv(emitter, "npageslabs_nonhuge", emitter_type_size, - &npageslabs_nonhuge); - emitter_json_kv( - emitter, "nactive_nonhuge", emitter_type_size, &nactive_nonhuge); - emitter_json_kv( - emitter, "ndirty_nonhuge", emitter_type_size, &ndirty_nonhuge); - emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); - emitter_json_object_end(emitter); /* End "empty_slabs" */ - - /* Last, nonfull slab stats. */ COL_HDR(row, size, NULL, right, 20, size) COL_HDR(row, ind, NULL, right, 4, unsigned) COL_HDR(row, npageslabs_huge, NULL, right, 16, size) @@ -1167,75 +1373,85 @@ stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) { COL_HDR(row, ndirty_nonhuge, NULL, right, 20, size) COL_HDR(row, nretained_nonhuge, NULL, right, 20, size) + emitter_table_printf(emitter, "pageslabs:\n"); + emitter_table_row(emitter, &header_row); + + /* + * Full and empty pageslabs are shown as two symbolic rows (size = + * "full"/"empty", no size-class index) at the top of the table; their + * JSON stays in the separate full_slabs / empty_slabs objects. + */ + col_ind.type = emitter_type_title; + col_ind.str_val = "-"; + col_size.type = emitter_type_title; + + stats_arena_hpa_slab_t sfull; + stats_gather_arena_hpa_slab(i, "full_slabs", &sfull); + stats_emit_arena_hpa_slab_json(emitter, &sfull, "full_slabs"); + col_size.str_val = "full"; + stats_hpa_slab_cols_set(&sfull, &col_npageslabs_huge, &col_nactive_huge, + &col_ndirty_huge, &col_npageslabs_nonhuge, &col_nactive_nonhuge, + &col_ndirty_nonhuge, &col_nretained_nonhuge); + emitter_table_row(emitter, &row); + + stats_arena_hpa_slab_t sempty; + stats_gather_arena_hpa_slab(i, "empty_slabs", &sempty); + stats_emit_arena_hpa_slab_json(emitter, &sempty, "empty_slabs"); + col_size.str_val = "empty"; + stats_hpa_slab_cols_set(&sempty, &col_npageslabs_huge, &col_nactive_huge, + &col_ndirty_huge, &col_npageslabs_nonhuge, &col_nactive_nonhuge, + &col_ndirty_nonhuge, &col_nretained_nonhuge); + emitter_table_row(emitter, &row); + + /* Numeric size/ind for the per-size-class rows below. */ + col_size.type = emitter_type_size; + col_ind.type = emitter_type_unsigned; + size_t stats_arenas_mib[CTL_MAX_DEPTH]; CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas"); stats_arenas_mib[2] = i; CTL_LEAF_PREPARE(stats_arenas_mib, 3, "hpa_shard.nonfull_slabs"); - emitter_table_printf(emitter, " In nonfull slabs:\n"); - emitter_table_row(emitter, &header_row); emitter_json_array_kv_begin(emitter, "nonfull_slabs"); - bool in_gap = false; + emitter_table_sparse_begin(emitter); for (pszind_t j = 0; j < PSSET_NPSIZES && j < SC_NPSIZES; j++) { - stats_arenas_mib[5] = j; + stats_arena_hpa_slab_t s; + stats_gather_arena_hpa_nonfull(stats_arenas_mib, j, &s); - CTL_LEAF(stats_arenas_mib, 6, "npageslabs_huge", - &npageslabs_huge, size_t); - CTL_LEAF( - stats_arenas_mib, 6, "nactive_huge", &nactive_huge, size_t); - CTL_LEAF( - stats_arenas_mib, 6, "ndirty_huge", &ndirty_huge, size_t); + bool is_gap = (s.npageslabs_huge == 0 && s.npageslabs_nonhuge == 0); - CTL_LEAF(stats_arenas_mib, 6, "npageslabs_nonhuge", - &npageslabs_nonhuge, size_t); - CTL_LEAF(stats_arenas_mib, 6, "nactive_nonhuge", - &nactive_nonhuge, size_t); - CTL_LEAF(stats_arenas_mib, 6, "ndirty_nonhuge", &ndirty_nonhuge, - size_t); - nretained_nonhuge = npageslabs_nonhuge * HUGEPAGE_PAGES - - nactive_nonhuge - ndirty_nonhuge; - - bool in_gap_prev = in_gap; - in_gap = (npageslabs_huge == 0 && npageslabs_nonhuge == 0); - if (in_gap_prev && !in_gap) { - emitter_table_printf( - emitter, " ---\n"); - } - - col_size.size_val = sz_pind2sz(j); - col_ind.size_val = j; - col_npageslabs_huge.size_val = npageslabs_huge; - col_nactive_huge.size_val = nactive_huge; - col_ndirty_huge.size_val = ndirty_huge; - col_npageslabs_nonhuge.size_val = npageslabs_nonhuge; - col_nactive_nonhuge.size_val = nactive_nonhuge; - col_ndirty_nonhuge.size_val = ndirty_nonhuge; - col_nretained_nonhuge.size_val = nretained_nonhuge; - if (!in_gap) { - emitter_table_row(emitter, &row); - } + char size_buf[48]; + stats_size_col_set(&col_size, sz_pind2sz(j), + j > 0 ? sz_pind2sz(j - 1) : 0, size_buf, sizeof(size_buf)); + col_ind.unsigned_val = j; + col_npageslabs_huge.size_val = s.npageslabs_huge; + col_nactive_huge.size_val = s.nactive_huge; + col_ndirty_huge.size_val = s.ndirty_huge; + col_npageslabs_nonhuge.size_val = s.npageslabs_nonhuge; + col_nactive_nonhuge.size_val = s.nactive_nonhuge; + col_ndirty_nonhuge.size_val = s.ndirty_nonhuge; + col_nretained_nonhuge.size_val = s.nretained_nonhuge; + emitter_table_sparse_row(emitter, &row, is_gap); emitter_json_object_begin(emitter); emitter_json_kv(emitter, "npageslabs_huge", emitter_type_size, - &npageslabs_huge); + &s.npageslabs_huge); emitter_json_kv( - emitter, "nactive_huge", emitter_type_size, &nactive_huge); + emitter, "nactive_huge", emitter_type_size, &s.nactive_huge); emitter_json_kv( - emitter, "ndirty_huge", emitter_type_size, &ndirty_huge); + emitter, "ndirty_huge", emitter_type_size, &s.ndirty_huge); emitter_json_kv(emitter, "npageslabs_nonhuge", - emitter_type_size, &npageslabs_nonhuge); + emitter_type_size, &s.npageslabs_nonhuge); emitter_json_kv(emitter, "nactive_nonhuge", emitter_type_size, - &nactive_nonhuge); + &s.nactive_nonhuge); emitter_json_kv(emitter, "ndirty_nonhuge", emitter_type_size, - &ndirty_nonhuge); + &s.ndirty_nonhuge); emitter_json_kv(emitter, "nretained_nonhuge", emitter_type_size, - &nretained_nonhuge); + &s.nretained_nonhuge); emitter_json_object_end(emitter); } emitter_json_array_end(emitter); /* End "nonfull_slabs" */ - if (in_gap) { - emitter_table_printf(emitter, " ---\n"); - } + emitter_table_sparse_end(emitter); } static void @@ -1250,6 +1466,11 @@ stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) { static void stats_arena_mutexes_print( emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { + /* + * Mutex readers gather counters directly into emitter columns, coupling + * collection to the output representation. Emission remains a separate + * mutex_stats_emit() call. + */ emitter_row_t row; emitter_col_t col_name; emitter_col_t col64[mutex_prof_num_uint64_t_counters]; @@ -1270,7 +1491,7 @@ stats_arena_mutexes_print( i++) { const char *name = arena_mutex_names[i]; emitter_json_object_kv_begin(emitter, name); - mutex_stats_read_arena( + mutex_stats_read( stats_arenas_mib, 4, name, &col_name, col64, col32, uptime); mutex_stats_emit(emitter, &row, col64, col32); emitter_json_object_end(emitter); /* Close the mutex dict. */ @@ -1278,90 +1499,57 @@ stats_arena_mutexes_print( emitter_json_object_end(emitter); /* End "mutexes". */ } -JEMALLOC_COLD static void -stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, - bool mutex, bool extents, bool hpa) { - char name[ARENA_NAME_LEN]; - char *namep = name; - unsigned nthreads; - const char *dss; - ssize_t dirty_decay_ms, muzzy_decay_ms; - size_t page, pactive, pdirty, pmuzzy, mapped, retained, pinned; - size_t base, internal, resident, metadata_edata, metadata_rtree, - metadata_thp, extent_avail; - uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; - uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; - size_t small_allocated; - uint64_t small_nmalloc, small_ndalloc, small_nrequests, small_nfills, - small_nflushes; - size_t large_allocated; - uint64_t large_nmalloc, large_ndalloc, large_nrequests, large_nfills, - large_nflushes; - size_t tcache_bytes, tcache_stashed_bytes, abandoned_vm; - uint64_t uptime; - - CTL_GET("arenas.page", &page, size_t); - if (i != MALLCTL_ARENAS_ALL && i != MALLCTL_ARENAS_DESTROYED) { - CTL_M1_GET("arena.0.name", i, (void *)&namep, const char *); +stats_emit_arena_basics(emitter_t *emitter, const stats_arena_basics_t *b) { + if (b->has_name) { + const char *namep = b->name; emitter_kv( emitter, "name", "name", emitter_type_string, &namep); } - - CTL_M2_GET("stats.arenas.0.nthreads", i, &nthreads, unsigned); emitter_kv(emitter, "nthreads", "assigned threads", - emitter_type_unsigned, &nthreads); - - CTL_M2_GET("stats.arenas.0.uptime", i, &uptime, uint64_t); - emitter_kv( - emitter, "uptime_ns", "uptime", emitter_type_uint64, &uptime); - - CTL_M2_GET("stats.arenas.0.dss", i, &dss, const char *); + emitter_type_unsigned, &b->nthreads); + emitter_kv(emitter, "uptime_ns", "uptime", emitter_type_uint64, + &b->uptime); emitter_kv(emitter, "dss", "dss allocation precedence", - emitter_type_string, &dss); + emitter_type_string, &b->dss); +} - CTL_M2_GET( - "stats.arenas.0.dirty_decay_ms", i, &dirty_decay_ms, ssize_t); - CTL_M2_GET( - "stats.arenas.0.muzzy_decay_ms", i, &muzzy_decay_ms, ssize_t); - CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t); - CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t); - CTL_M2_GET("stats.arenas.0.pmuzzy", i, &pmuzzy, size_t); - CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &dirty_npurge, uint64_t); - CTL_M2_GET( - "stats.arenas.0.dirty_nmadvise", i, &dirty_nmadvise, uint64_t); - CTL_M2_GET("stats.arenas.0.dirty_purged", i, &dirty_purged, uint64_t); - CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &muzzy_npurge, uint64_t); - CTL_M2_GET( - "stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise, uint64_t); - CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &muzzy_purged, uint64_t); +static uint64_t +stats_arena_basics_print(emitter_t *emitter, unsigned i) { + stats_arena_basics_t basics; + stats_gather_arena_basics(i, &basics); + stats_emit_arena_basics(emitter, &basics); + return basics.uptime; +} +static void +stats_emit_arena_decay(emitter_t *emitter, const stats_arena_decay_t *d) { emitter_row_t decay_row; emitter_row_init(&decay_row); /* JSON-style emission. */ emitter_json_kv( - emitter, "dirty_decay_ms", emitter_type_ssize, &dirty_decay_ms); + emitter, "dirty_decay_ms", emitter_type_ssize, &d->dirty_decay_ms); emitter_json_kv( - emitter, "muzzy_decay_ms", emitter_type_ssize, &muzzy_decay_ms); + emitter, "muzzy_decay_ms", emitter_type_ssize, &d->muzzy_decay_ms); - emitter_json_kv(emitter, "pactive", emitter_type_size, &pactive); - emitter_json_kv(emitter, "pdirty", emitter_type_size, &pdirty); - emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &pmuzzy); + emitter_json_kv(emitter, "pactive", emitter_type_size, &d->pactive); + emitter_json_kv(emitter, "pdirty", emitter_type_size, &d->pdirty); + emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &d->pmuzzy); emitter_json_kv( - emitter, "dirty_npurge", emitter_type_uint64, &dirty_npurge); + emitter, "dirty_npurge", emitter_type_uint64, &d->dirty_npurge); emitter_json_kv( - emitter, "dirty_nmadvise", emitter_type_uint64, &dirty_nmadvise); + emitter, "dirty_nmadvise", emitter_type_uint64, &d->dirty_nmadvise); emitter_json_kv( - emitter, "dirty_purged", emitter_type_uint64, &dirty_purged); + emitter, "dirty_purged", emitter_type_uint64, &d->dirty_purged); emitter_json_kv( - emitter, "muzzy_npurge", emitter_type_uint64, &muzzy_npurge); + emitter, "muzzy_npurge", emitter_type_uint64, &d->muzzy_npurge); emitter_json_kv( - emitter, "muzzy_nmadvise", emitter_type_uint64, &muzzy_nmadvise); + emitter, "muzzy_nmadvise", emitter_type_uint64, &d->muzzy_nmadvise); emitter_json_kv( - emitter, "muzzy_purged", emitter_type_uint64, &muzzy_purged); + emitter, "muzzy_purged", emitter_type_uint64, &d->muzzy_purged); /* Table-style emission. */ COL(decay_row, decay_type, right, 9, title); @@ -1388,54 +1576,65 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, /* Dirty row. */ col_decay_type.str_val = "dirty:"; - if (dirty_decay_ms >= 0) { + if (d->dirty_decay_ms >= 0) { col_decay_time.type = emitter_type_ssize; - col_decay_time.ssize_val = dirty_decay_ms; + col_decay_time.ssize_val = d->dirty_decay_ms; } else { col_decay_time.type = emitter_type_title; col_decay_time.str_val = "N/A"; } col_decay_npages.type = emitter_type_size; - col_decay_npages.size_val = pdirty; + col_decay_npages.size_val = d->pdirty; col_decay_sweeps.type = emitter_type_uint64; - col_decay_sweeps.uint64_val = dirty_npurge; + col_decay_sweeps.uint64_val = d->dirty_npurge; col_decay_madvises.type = emitter_type_uint64; - col_decay_madvises.uint64_val = dirty_nmadvise; + col_decay_madvises.uint64_val = d->dirty_nmadvise; col_decay_purged.type = emitter_type_uint64; - col_decay_purged.uint64_val = dirty_purged; + col_decay_purged.uint64_val = d->dirty_purged; emitter_table_row(emitter, &decay_row); /* Muzzy row. */ col_decay_type.str_val = "muzzy:"; - if (muzzy_decay_ms >= 0) { + if (d->muzzy_decay_ms >= 0) { col_decay_time.type = emitter_type_ssize; - col_decay_time.ssize_val = muzzy_decay_ms; + col_decay_time.ssize_val = d->muzzy_decay_ms; } else { col_decay_time.type = emitter_type_title; col_decay_time.str_val = "N/A"; } col_decay_npages.type = emitter_type_size; - col_decay_npages.size_val = pmuzzy; + col_decay_npages.size_val = d->pmuzzy; col_decay_sweeps.type = emitter_type_uint64; - col_decay_sweeps.uint64_val = muzzy_npurge; + col_decay_sweeps.uint64_val = d->muzzy_npurge; col_decay_madvises.type = emitter_type_uint64; - col_decay_madvises.uint64_val = muzzy_nmadvise; + col_decay_madvises.uint64_val = d->muzzy_nmadvise; col_decay_purged.type = emitter_type_uint64; - col_decay_purged.uint64_val = muzzy_purged; + col_decay_purged.uint64_val = d->muzzy_purged; emitter_table_row(emitter, &decay_row); +} - /* Small / large / total allocation counts. */ +static size_t +stats_arena_decay_print(emitter_t *emitter, unsigned i) { + stats_arena_decay_t decay; + stats_gather_arena_decay(i, &decay); + stats_emit_arena_decay(emitter, &decay); + return decay.pactive; +} + +static void +stats_emit_arena_alloc(emitter_t *emitter, const stats_arena_alloc_t *small, + const stats_arena_alloc_t *large, uint64_t uptime) { emitter_row_t alloc_count_row; emitter_row_init(&alloc_count_row); @@ -1478,70 +1677,78 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, col_count_nfills_ps.type = emitter_type_uint64; col_count_nflushes_ps.type = emitter_type_uint64; -#define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ - CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ - &small_or_large##_##name, valtype##_t); \ - emitter_json_kv( \ - emitter, #name, emitter_type_##valtype, &small_or_large##_##name); \ + /* + * JSON keys for the value columns; emitter_row() emits these as kvs. + * The (#/sec) rate columns and the title column stay table-only (no + * json_key). + */ + col_count_allocated.json_key = "allocated"; + col_count_nmalloc.json_key = "nmalloc"; + col_count_ndalloc.json_key = "ndalloc"; + col_count_nrequests.json_key = "nrequests"; + col_count_nfills.json_key = "nfills"; + col_count_nflushes.json_key = "nflushes"; + +#define EMIT_ALLOC_STAT(a, name, valtype) \ col_count_##name.type = emitter_type_##valtype; \ - col_count_##name.valtype##_val = small_or_large##_##name; + col_count_##name.valtype##_val = (a)->name; emitter_json_object_kv_begin(emitter, "small"); col_count_title.str_val = "small:"; - GET_AND_EMIT_ALLOC_STAT(small, allocated, size) - GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64) + EMIT_ALLOC_STAT(small, allocated, size) + EMIT_ALLOC_STAT(small, nmalloc, uint64) col_count_nmalloc_ps.uint64_val = rate_per_second( col_count_nmalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64) + EMIT_ALLOC_STAT(small, ndalloc, uint64) col_count_ndalloc_ps.uint64_val = rate_per_second( col_count_ndalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64) + EMIT_ALLOC_STAT(small, nrequests, uint64) col_count_nrequests_ps.uint64_val = rate_per_second( col_count_nrequests.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nfills, uint64) + EMIT_ALLOC_STAT(small, nfills, uint64) col_count_nfills_ps.uint64_val = rate_per_second( col_count_nfills.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(small, nflushes, uint64) + EMIT_ALLOC_STAT(small, nflushes, uint64) col_count_nflushes_ps.uint64_val = rate_per_second( col_count_nflushes.uint64_val, uptime); - emitter_table_row(emitter, &alloc_count_row); + emitter_row(emitter, &alloc_count_row); emitter_json_object_end(emitter); /* Close "small". */ emitter_json_object_kv_begin(emitter, "large"); col_count_title.str_val = "large:"; - GET_AND_EMIT_ALLOC_STAT(large, allocated, size) - GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64) + EMIT_ALLOC_STAT(large, allocated, size) + EMIT_ALLOC_STAT(large, nmalloc, uint64) col_count_nmalloc_ps.uint64_val = rate_per_second( col_count_nmalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64) + EMIT_ALLOC_STAT(large, ndalloc, uint64) col_count_ndalloc_ps.uint64_val = rate_per_second( col_count_ndalloc.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64) + EMIT_ALLOC_STAT(large, nrequests, uint64) col_count_nrequests_ps.uint64_val = rate_per_second( col_count_nrequests.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nfills, uint64) + EMIT_ALLOC_STAT(large, nfills, uint64) col_count_nfills_ps.uint64_val = rate_per_second( col_count_nfills.uint64_val, uptime); - GET_AND_EMIT_ALLOC_STAT(large, nflushes, uint64) + EMIT_ALLOC_STAT(large, nflushes, uint64) col_count_nflushes_ps.uint64_val = rate_per_second( col_count_nflushes.uint64_val, uptime); - emitter_table_row(emitter, &alloc_count_row); + emitter_row(emitter, &alloc_count_row); emitter_json_object_end(emitter); /* Close "large". */ -#undef GET_AND_EMIT_ALLOC_STAT +#undef EMIT_ALLOC_STAT /* Aggregated small + large stats are emitter only in table mode. */ col_count_title.str_val = "total:"; - col_count_allocated.size_val = small_allocated + large_allocated; - col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; - col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; - col_count_nrequests.uint64_val = small_nrequests + large_nrequests; - col_count_nfills.uint64_val = small_nfills + large_nfills; - col_count_nflushes.uint64_val = small_nflushes + large_nflushes; + col_count_allocated.size_val = small->allocated + large->allocated; + col_count_nmalloc.uint64_val = small->nmalloc + large->nmalloc; + col_count_ndalloc.uint64_val = small->ndalloc + large->ndalloc; + col_count_nrequests.uint64_val = small->nrequests + large->nrequests; + col_count_nfills.uint64_val = small->nfills + large->nfills; + col_count_nflushes.uint64_val = small->nflushes + large->nflushes; col_count_nmalloc_ps.uint64_val = rate_per_second( col_count_nmalloc.uint64_val, uptime); col_count_ndalloc_ps.uint64_val = rate_per_second( @@ -1553,7 +1760,18 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, col_count_nflushes_ps.uint64_val = rate_per_second( col_count_nflushes.uint64_val, uptime); emitter_table_row(emitter, &alloc_count_row); +} +static void +stats_arena_alloc_print(emitter_t *emitter, unsigned i, uint64_t uptime) { + stats_arena_alloc_t small_alloc, large_alloc; + stats_gather_arena_alloc(i, &small_alloc, &large_alloc); + stats_emit_arena_alloc(emitter, &small_alloc, &large_alloc, uptime); +} + +static void +stats_emit_arena_mem(emitter_t *emitter, const stats_arena_mem_t *mem, + size_t active_bytes) { emitter_row_t mem_count_row; emitter_row_init(&mem_count_row); @@ -1571,35 +1789,59 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, mem_count_val.type = emitter_type_title; mem_count_val.str_val = ""; - emitter_table_row(emitter, &mem_count_row); + /* Blank spacer row (both columns are empty titles; table only). */ + emitter_row(emitter, &mem_count_row); mem_count_val.type = emitter_type_size; - /* Active count in bytes is emitted only in table mode. */ + /* + * Active count in bytes is table only: mem_count_val carries no + * json_key, so emitter_row() skips it in JSON. + */ mem_count_title.str_val = "active:"; - mem_count_val.size_val = pactive * page; - emitter_table_row(emitter, &mem_count_row); + mem_count_val.size_val = active_bytes; + emitter_row(emitter, &mem_count_row); -#define GET_AND_EMIT_MEM_STAT(stat) \ - CTL_M2_GET("stats.arenas.0." #stat, i, &stat, size_t); \ - emitter_json_kv(emitter, #stat, emitter_type_size, &stat); \ + /* + * Each remaining stat is one row: the value column's json_key makes it a + * "": value pair in JSON and the aligned value in the table. + */ +#define EMIT_MEM_STAT(stat) \ mem_count_title.str_val = #stat ":"; \ - mem_count_val.size_val = stat; \ - emitter_table_row(emitter, &mem_count_row); + mem_count_val.json_key = #stat; \ + mem_count_val.size_val = mem->stat; \ + emitter_row(emitter, &mem_count_row); - GET_AND_EMIT_MEM_STAT(mapped) - GET_AND_EMIT_MEM_STAT(retained) - GET_AND_EMIT_MEM_STAT(pinned) - GET_AND_EMIT_MEM_STAT(base) - GET_AND_EMIT_MEM_STAT(internal) - GET_AND_EMIT_MEM_STAT(metadata_edata) - GET_AND_EMIT_MEM_STAT(metadata_rtree) - GET_AND_EMIT_MEM_STAT(metadata_thp) - GET_AND_EMIT_MEM_STAT(tcache_bytes) - GET_AND_EMIT_MEM_STAT(tcache_stashed_bytes) - GET_AND_EMIT_MEM_STAT(resident) - GET_AND_EMIT_MEM_STAT(abandoned_vm) - GET_AND_EMIT_MEM_STAT(extent_avail) -#undef GET_AND_EMIT_MEM_STAT + EMIT_MEM_STAT(mapped) + EMIT_MEM_STAT(retained) + EMIT_MEM_STAT(pinned) + EMIT_MEM_STAT(base) + EMIT_MEM_STAT(internal) + EMIT_MEM_STAT(metadata_edata) + EMIT_MEM_STAT(metadata_rtree) + EMIT_MEM_STAT(metadata_thp) + EMIT_MEM_STAT(tcache_bytes) + EMIT_MEM_STAT(tcache_stashed_bytes) + EMIT_MEM_STAT(resident) + EMIT_MEM_STAT(abandoned_vm) + EMIT_MEM_STAT(extent_avail) +#undef EMIT_MEM_STAT +} + +static void +stats_arena_mem_print(emitter_t *emitter, unsigned i, size_t pactive) { + stats_arena_mem_t m; + stats_gather_arena_mem(i, &m); + stats_emit_arena_mem(emitter, &m, pactive * m.page); +} + +JEMALLOC_COLD +static void +stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, + bool mutex, bool extents, bool hpa) { + uint64_t uptime = stats_arena_basics_print(emitter, i); + size_t pactive = stats_arena_decay_print(emitter, i); + stats_arena_alloc_print(emitter, i, uptime); + stats_arena_mem_print(emitter, i, pactive); if (opt_pac_sec_opts.nshards > 0) { stats_arena_pac_sec_print(emitter, i); @@ -1622,9 +1864,68 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, } } -JEMALLOC_COLD static void -stats_general_print(emitter_t *emitter) { +stats_general_version(emitter_t *emitter) { + const char *cpv; + + CTL_GET("version", &cpv, const char *); + emitter_kv(emitter, "version", "Version", emitter_type_string, &cpv); +} + +static void +stats_general_config(emitter_t *emitter) { + /* + * Most config entries are independent scalars that + * CONFIG_WRITE_BOOL reads and emits together. Keeping those operations + * local is clearer than introducing a gather struct for this flat list. + */ + bool bv; + + emitter_dict_begin(emitter, "config", "Build-time option settings"); + CONFIG_WRITE_BOOL(cache_oblivious); + CONFIG_WRITE_BOOL(debug); + CONFIG_WRITE_BOOL(fill); + CONFIG_WRITE_BOOL(infallible_new); + CONFIG_WRITE_BOOL(lazy_lock); + emitter_kv(emitter, "malloc_conf", "config.malloc_conf", + emitter_type_string, &config_malloc_conf); + + CONFIG_WRITE_BOOL(opt_safety_checks); + CONFIG_WRITE_BOOL(prof); + CONFIG_WRITE_BOOL(prof_libgcc); + CONFIG_WRITE_BOOL(prof_libunwind); + CONFIG_WRITE_BOOL(prof_frameptr); + CONFIG_WRITE_BOOL(stats); + CONFIG_WRITE_BOOL(utrace); + CONFIG_WRITE_BOOL(xmalloc); + emitter_dict_end(emitter); /* Close "config" dict. */ +} + +static void +stats_general_system(emitter_t *emitter) { + emitter_dict_begin(emitter, "system", "System configuration"); + + /* + * This shows system's THP mode detected at jemalloc's init time. + * jemalloc does not re-detect the mode even if it changes after + * jemalloc's init. It is assumed that system's THP mode is stable + * during the process's lifetime and a violation could lead to + * undefined behavior. + */ + const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode]; + emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string, + &thp_mode_name); + + emitter_dict_end(emitter); /* Close "system". */ +} + +static void +stats_general_opts(emitter_t *emitter) { + /* + * The OPT_WRITE_* macros conditionally read and emit most options. + * Options needing special formatting or availability checks stay + * explicit below. + */ const char *cpv; bool bv, bv2; unsigned uv; @@ -1643,82 +1944,6 @@ stats_general_print(emitter_t *emitter) { i64sz = sizeof(int64_t); u64sz = sizeof(uint64_t); - CTL_GET("version", &cpv, const char *); - emitter_kv(emitter, "version", "Version", emitter_type_string, &cpv); - - /* config. */ - emitter_dict_begin(emitter, "config", "Build-time option settings"); -#define CONFIG_WRITE_BOOL(name) \ - do { \ - CTL_GET("config." #name, &bv, bool); \ - emitter_kv( \ - emitter, #name, "config." #name, emitter_type_bool, &bv); \ - } while (0) - - CONFIG_WRITE_BOOL(cache_oblivious); - CONFIG_WRITE_BOOL(debug); - CONFIG_WRITE_BOOL(fill); - CONFIG_WRITE_BOOL(infallible_new); - CONFIG_WRITE_BOOL(lazy_lock); - emitter_kv(emitter, "malloc_conf", "config.malloc_conf", - emitter_type_string, &config_malloc_conf); - - CONFIG_WRITE_BOOL(opt_safety_checks); - CONFIG_WRITE_BOOL(prof); - CONFIG_WRITE_BOOL(prof_libgcc); - CONFIG_WRITE_BOOL(prof_libunwind); - CONFIG_WRITE_BOOL(prof_frameptr); - CONFIG_WRITE_BOOL(stats); - CONFIG_WRITE_BOOL(utrace); - CONFIG_WRITE_BOOL(xmalloc); -#undef CONFIG_WRITE_BOOL - emitter_dict_end(emitter); /* Close "config" dict. */ - - /* system. */ - emitter_dict_begin(emitter, "system", "System configuration"); - - /* - * This shows system's THP mode detected at jemalloc's init time. - * jemalloc does not re-detect the mode even if it changes after - * jemalloc's init. It is assumed that system's THP mode is stable - * during the process's lifetime and a violation could lead to - * undefined behavior. - */ - const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode]; - emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string, - &thp_mode_name); - - emitter_dict_end(emitter); /* Close "system". */ - - /* opt. */ -#define OPT_WRITE(name, var, size, emitter_type) \ - if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \ - emitter_kv(emitter, name, "opt." name, emitter_type, &var); \ - } - -#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, altname) \ - if (je_mallctl("opt." name, (void *)&var1, &size, NULL, 0) == 0 \ - && je_mallctl(altname, (void *)&var2, &size, NULL, 0) == 0) { \ - emitter_kv_note(emitter, name, "opt." name, emitter_type, \ - &var1, altname, emitter_type, &var2); \ - } - -#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool) -#define OPT_WRITE_BOOL_MUTABLE(name, altname) \ - OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname) - -#define OPT_WRITE_UNSIGNED(name) OPT_WRITE(name, uv, usz, emitter_type_unsigned) - -#define OPT_WRITE_INT64(name) OPT_WRITE(name, i64v, i64sz, emitter_type_int64) -#define OPT_WRITE_UINT64(name) OPT_WRITE(name, u64v, u64sz, emitter_type_uint64) - -#define OPT_WRITE_SIZE_T(name) OPT_WRITE(name, sv, ssz, emitter_type_size) -#define OPT_WRITE_SSIZE_T(name) OPT_WRITE(name, ssv, sssz, emitter_type_ssize) -#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ - OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, altname) - -#define OPT_WRITE_CHAR_P(name) OPT_WRITE(name, cpv, cpsz, emitter_type_string) - emitter_dict_begin(emitter, "opt", "Run-time option settings"); /* @@ -1840,8 +2065,6 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("prof_leak_error") OPT_WRITE_BOOL("stats_print") OPT_WRITE_CHAR_P("stats_print_opts") - OPT_WRITE_BOOL("stats_print") - OPT_WRITE_CHAR_P("stats_print_opts") OPT_WRITE_INT64("stats_interval") OPT_WRITE_CHAR_P("stats_interval_opts") OPT_WRITE_CHAR_P("zero_realloc") @@ -1849,18 +2072,15 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("disable_large_size_classes") emitter_dict_end(emitter); /* Close "opt". */ +} -#undef OPT_WRITE -#undef OPT_WRITE_MUTABLE -#undef OPT_WRITE_BOOL -#undef OPT_WRITE_BOOL_MUTABLE -#undef OPT_WRITE_UNSIGNED -#undef OPT_WRITE_SSIZE_T -#undef OPT_WRITE_SSIZE_T_MUTABLE -#undef OPT_WRITE_CHAR_P - - /* prof. */ +static void +stats_general_prof(emitter_t *emitter) { if (config_prof) { + bool bv; + uint64_t u64v; + ssize_t ssv; + emitter_dict_begin(emitter, "prof", "Profiling settings"); CTL_GET("prof.thread_active_init", &bv, bool); @@ -1885,105 +2105,101 @@ stats_general_print(emitter_t *emitter) { emitter_dict_end(emitter); /* Close "prof". */ } +} + +static void +stats_general_bin_meta_print(emitter_t *emitter, unsigned nbins) { + /* + * Streaming JSON-only size-class metadata table (per-row gather+emit); + * see stats_arena_bins_print for the streaming rationale. + */ + emitter_json_array_kv_begin(emitter, "bin"); + size_t mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(mib, 0, "arenas.bin"); + for (unsigned i = 0; i < nbins; i++) { + stats_arena_bin_meta_t bm; + stats_gather_arena_bin_meta(mib, i, &bm); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "size", emitter_type_size, &bm.size); + emitter_json_kv(emitter, "nregs", emitter_type_uint32, &bm.nregs); + emitter_json_kv( + emitter, "slab_size", emitter_type_size, &bm.slab_size); + emitter_json_kv(emitter, "nshards", emitter_type_uint32, &bm.nshards); + emitter_json_object_end(emitter); + } + emitter_json_array_end(emitter); /* Close "bin". */ +} + +static void +stats_general_lextent_meta_print(emitter_t *emitter, unsigned nlextents) { + /* Streaming JSON-only metadata table; see stats_general_bin_meta_print. */ + emitter_json_array_kv_begin(emitter, "lextent"); + size_t mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(mib, 0, "arenas.lextent"); + for (unsigned i = 0; i < nlextents; i++) { + stats_arena_lextent_meta_t lm; + stats_gather_arena_lextent_meta(mib, i, &lm); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "size", emitter_type_size, &lm.size); + emitter_json_object_end(emitter); + } + emitter_json_array_end(emitter); /* Close "lextent". */ +} + +static void +stats_general_arenas_print(emitter_t *emitter, bool omit_size_class_meta) { + stats_arena_config_t cfg; + stats_gather_arena_config(&cfg); - /* arenas. */ /* * The json output sticks arena info into an "arenas" dict; the table * output puts them at the top-level. */ emitter_json_object_kv_begin(emitter, "arenas"); - CTL_GET("arenas.narenas", &uv, unsigned); - emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv); - - /* - * Decay settings are emitted only in json mode; in table mode, they're - * emitted as notes with the opt output, above. - */ - CTL_GET("arenas.dirty_decay_ms", &ssv, ssize_t); - emitter_json_kv(emitter, "dirty_decay_ms", emitter_type_ssize, &ssv); - - CTL_GET("arenas.muzzy_decay_ms", &ssv, ssize_t); - emitter_json_kv(emitter, "muzzy_decay_ms", emitter_type_ssize, &ssv); - - CTL_GET("arenas.quantum", &sv, size_t); - emitter_kv(emitter, "quantum", "Quantum size", emitter_type_size, &sv); - - CTL_GET("arenas.page", &sv, size_t); - emitter_kv(emitter, "page", "Page size", emitter_type_size, &sv); - - CTL_GET("arenas.hugepage", &sv, size_t); emitter_kv( - emitter, "hugepage", "Hugepage size", emitter_type_size, &sv); - - if (je_mallctl("arenas.tcache_max", (void *)&sv, &ssz, NULL, 0) == 0) { - emitter_kv(emitter, "tcache_max", - "Maximum thread-cached size class", emitter_type_size, &sv); - } - - unsigned arenas_nbins; - CTL_GET("arenas.nbins", &arenas_nbins, unsigned); - emitter_kv(emitter, "nbins", "Number of bin size classes", - emitter_type_unsigned, &arenas_nbins); - - unsigned arenas_nhbins; - CTL_GET("arenas.nhbins", &arenas_nhbins, unsigned); - emitter_kv(emitter, "nhbins", "Number of thread-cache bin size classes", - emitter_type_unsigned, &arenas_nhbins); + emitter, "narenas", "Arenas", emitter_type_unsigned, &cfg.narenas); /* - * We do enough mallctls in a loop that we actually want to omit them - * (not just omit the printing). + * Decay settings are emitted only in json mode; in table mode, they + * are emitted as notes with the opt output, above. */ - if (emitter_outputs_json(emitter)) { - emitter_json_array_kv_begin(emitter, "bin"); - size_t arenas_bin_mib[CTL_MAX_DEPTH]; - CTL_LEAF_PREPARE(arenas_bin_mib, 0, "arenas.bin"); - for (unsigned i = 0; i < arenas_nbins; i++) { - arenas_bin_mib[2] = i; - emitter_json_object_begin(emitter); + emitter_json_kv( + emitter, "dirty_decay_ms", emitter_type_ssize, &cfg.dirty_decay_ms); + emitter_json_kv( + emitter, "muzzy_decay_ms", emitter_type_ssize, &cfg.muzzy_decay_ms); - CTL_LEAF(arenas_bin_mib, 3, "size", &sv, size_t); - emitter_json_kv( - emitter, "size", emitter_type_size, &sv); + emitter_kv(emitter, "quantum", "Quantum size", emitter_type_size, + &cfg.quantum); + emitter_kv(emitter, "page", "Page size", emitter_type_size, &cfg.page); + emitter_kv(emitter, "hugepage", "Hugepage size", emitter_type_size, + &cfg.hugepage); - CTL_LEAF(arenas_bin_mib, 3, "nregs", &u32v, uint32_t); - emitter_json_kv( - emitter, "nregs", emitter_type_uint32, &u32v); - - CTL_LEAF(arenas_bin_mib, 3, "slab_size", &sv, size_t); - emitter_json_kv( - emitter, "slab_size", emitter_type_size, &sv); - - CTL_LEAF(arenas_bin_mib, 3, "nshards", &u32v, uint32_t); - emitter_json_kv( - emitter, "nshards", emitter_type_uint32, &u32v); - - emitter_json_object_end(emitter); - } - emitter_json_array_end(emitter); /* Close "bin". */ + if (cfg.have_tcache_max) { + emitter_kv(emitter, "tcache_max", + "Maximum thread-cached size class", emitter_type_size, + &cfg.tcache_max); + } + + emitter_kv(emitter, "nbins", "Number of bin size classes", + emitter_type_unsigned, &cfg.nbins); + emitter_kv(emitter, "nhbins", "Number of thread-cache bin size classes", + emitter_type_unsigned, &cfg.nhbins); + + /* + * Per-size-class geometry is omitted from table output. It is present + * in JSON whenever general output is enabled; when omitted, it is not + * gathered either. + */ + if (!omit_size_class_meta) { + stats_general_bin_meta_print(emitter, cfg.nbins); } - unsigned nlextents; - CTL_GET("arenas.nlextents", &nlextents, unsigned); emitter_kv(emitter, "nlextents", "Number of large size classes", - emitter_type_unsigned, &nlextents); + emitter_type_unsigned, &cfg.nlextents); - if (emitter_outputs_json(emitter)) { - emitter_json_array_kv_begin(emitter, "lextent"); - size_t arenas_lextent_mib[CTL_MAX_DEPTH]; - CTL_LEAF_PREPARE(arenas_lextent_mib, 0, "arenas.lextent"); - for (unsigned i = 0; i < nlextents; i++) { - arenas_lextent_mib[2] = i; - emitter_json_object_begin(emitter); - - CTL_LEAF(arenas_lextent_mib, 3, "size", &sv, size_t); - emitter_json_kv( - emitter, "size", emitter_type_size, &sv); - - emitter_json_object_end(emitter); - } - emitter_json_array_end(emitter); /* Close "lextent". */ + if (!omit_size_class_meta) { + stats_general_lextent_meta_print(emitter, cfg.nlextents); } emitter_json_object_end(emitter); /* Close "arenas" */ @@ -1991,258 +2207,166 @@ stats_general_print(emitter_t *emitter) { JEMALLOC_COLD static void -stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, - bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) { - /* - * These should be deleted. We keep them around for a while, to aid in - * the transition to the emitter code. - */ - size_t allocated, active, metadata, metadata_edata, metadata_rtree, - metadata_thp, resident, mapped, retained, pinned; - size_t num_background_threads; - size_t zero_reallocs; - uint64_t background_thread_num_runs, background_thread_run_interval; - - CTL_GET("stats.allocated", &allocated, size_t); - CTL_GET("stats.active", &active, size_t); - CTL_GET("stats.metadata", &metadata, size_t); - CTL_GET("stats.metadata_edata", &metadata_edata, size_t); - CTL_GET("stats.metadata_rtree", &metadata_rtree, size_t); - CTL_GET("stats.metadata_thp", &metadata_thp, size_t); - CTL_GET("stats.resident", &resident, size_t); - CTL_GET("stats.mapped", &mapped, size_t); - CTL_GET("stats.retained", &retained, size_t); - CTL_GET("stats.pinned", &pinned, size_t); - - CTL_GET("stats.zero_reallocs", &zero_reallocs, size_t); - - if (have_background_thread) { - CTL_GET("stats.background_thread.num_threads", - &num_background_threads, size_t); - CTL_GET("stats.background_thread.num_runs", - &background_thread_num_runs, uint64_t); - CTL_GET("stats.background_thread.run_interval", - &background_thread_run_interval, uint64_t); - } else { - num_background_threads = 0; - background_thread_num_runs = 0; - background_thread_run_interval = 0; - } +stats_general_print(emitter_t *emitter, bool omit_size_class_meta) { + stats_general_version(emitter); + stats_general_config(emitter); + stats_general_system(emitter); + stats_general_opts(emitter); + stats_general_prof(emitter); + stats_general_arenas_print(emitter, omit_size_class_meta); +} +static void +stats_emit_global(emitter_t *emitter, const stats_global_t *g) { /* Generic global stats. */ - emitter_json_object_kv_begin(emitter, "stats"); - emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated); - emitter_json_kv(emitter, "active", emitter_type_size, &active); - emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata); + emitter_json_kv(emitter, "allocated", emitter_type_size, &g->allocated); + emitter_json_kv(emitter, "active", emitter_type_size, &g->active); + emitter_json_kv(emitter, "metadata", emitter_type_size, &g->metadata); emitter_json_kv( - emitter, "metadata_edata", emitter_type_size, &metadata_edata); + emitter, "metadata_edata", emitter_type_size, &g->metadata_edata); emitter_json_kv( - emitter, "metadata_rtree", emitter_type_size, &metadata_rtree); + emitter, "metadata_rtree", emitter_type_size, &g->metadata_rtree); emitter_json_kv( - emitter, "metadata_thp", emitter_type_size, &metadata_thp); - emitter_json_kv(emitter, "resident", emitter_type_size, &resident); - emitter_json_kv(emitter, "mapped", emitter_type_size, &mapped); - emitter_json_kv(emitter, "retained", emitter_type_size, &retained); - emitter_json_kv(emitter, "pinned", emitter_type_size, &pinned); + emitter, "metadata_thp", emitter_type_size, &g->metadata_thp); + emitter_json_kv(emitter, "resident", emitter_type_size, &g->resident); + emitter_json_kv(emitter, "mapped", emitter_type_size, &g->mapped); + emitter_json_kv(emitter, "retained", emitter_type_size, &g->retained); + emitter_json_kv(emitter, "pinned", emitter_type_size, &g->pinned); emitter_json_kv( - emitter, "zero_reallocs", emitter_type_size, &zero_reallocs); + emitter, "zero_reallocs", emitter_type_size, &g->zero_reallocs); emitter_table_printf(emitter, "Allocated: %zu, active: %zu, " "metadata: %zu (n_thp %zu, edata %zu, rtree %zu), resident: %zu, " "mapped: %zu, retained: %zu, pinned: %zu\n", - allocated, active, metadata, metadata_thp, metadata_edata, - metadata_rtree, resident, mapped, retained, pinned); + g->allocated, g->active, g->metadata, g->metadata_thp, + g->metadata_edata, g->metadata_rtree, g->resident, g->mapped, + g->retained, g->pinned); /* Strange behaviors */ emitter_table_printf(emitter, - "Count of realloc(non-null-ptr, 0) calls: %zu\n", zero_reallocs); + "Count of realloc(non-null-ptr, 0) calls: %zu\n", g->zero_reallocs); /* Background thread stats. */ emitter_json_object_kv_begin(emitter, "background_thread"); - emitter_json_kv( - emitter, "num_threads", emitter_type_size, &num_background_threads); + emitter_json_kv(emitter, "num_threads", emitter_type_size, + &g->num_background_threads); emitter_json_kv(emitter, "num_runs", emitter_type_uint64, - &background_thread_num_runs); + &g->background_thread_num_runs); emitter_json_kv(emitter, "run_interval", emitter_type_uint64, - &background_thread_run_interval); + &g->background_thread_run_interval); emitter_json_object_end(emitter); /* Close "background_thread". */ emitter_table_printf(emitter, "Background threads: %zu, " "num_runs: %" FMTu64 ", run_interval: %" FMTu64 " ns\n", - num_background_threads, background_thread_num_runs, - background_thread_run_interval); - - if (mutex) { - emitter_row_t row; - emitter_col_t name; - emitter_col_t col64[mutex_prof_num_uint64_t_counters]; - emitter_col_t col32[mutex_prof_num_uint32_t_counters]; - uint64_t uptime; - - emitter_row_init(&row); - mutex_stats_init_cols(&row, "", &name, col64, col32); - - emitter_table_row(emitter, &row); - emitter_json_object_kv_begin(emitter, "mutexes"); - - CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); - - size_t stats_mutexes_mib[CTL_MAX_DEPTH]; - CTL_LEAF_PREPARE(stats_mutexes_mib, 0, "stats.mutexes"); - for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { - mutex_stats_read_global(stats_mutexes_mib, 2, - global_mutex_names[i], &name, col64, col32, uptime); - emitter_json_object_kv_begin( - emitter, global_mutex_names[i]); - mutex_stats_emit(emitter, &row, col64, col32); - emitter_json_object_end(emitter); - } - - emitter_json_object_end(emitter); /* Close "mutexes". */ - } - - emitter_json_object_end(emitter); /* Close "stats". */ - - if (merged || destroyed || unmerged) { - unsigned narenas; - - emitter_json_object_kv_begin(emitter, "stats.arenas"); - - CTL_GET("arenas.narenas", &narenas, unsigned); - size_t mib[3]; - size_t miblen = sizeof(mib) / sizeof(size_t); - size_t sz; - VARIABLE_ARRAY_UNSAFE(bool, initialized, narenas); - bool destroyed_initialized; - unsigned i, ninitialized; - - xmallctlnametomib("arena.0.initialized", mib, &miblen); - for (i = ninitialized = 0; i < narenas; i++) { - mib[1] = i; - sz = sizeof(bool); - xmallctlbymib( - mib, miblen, &initialized[i], &sz, NULL, 0); - if (initialized[i]) { - ninitialized++; - } - } - mib[1] = MALLCTL_ARENAS_DESTROYED; - sz = sizeof(bool); - xmallctlbymib( - mib, miblen, &destroyed_initialized, &sz, NULL, 0); - - /* Merged stats. */ - if (merged && (ninitialized > 1 || !unmerged)) { - /* Print merged arena stats. */ - emitter_table_printf(emitter, "Merged arenas stats:\n"); - emitter_json_object_kv_begin(emitter, "merged"); - stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins, - large, mutex, extents, hpa); - emitter_json_object_end(emitter); /* Close "merged". */ - } - - /* Destroyed stats. */ - if (destroyed_initialized && destroyed) { - /* Print destroyed arena stats. */ - emitter_table_printf( - emitter, "Destroyed arenas stats:\n"); - emitter_json_object_kv_begin(emitter, "destroyed"); - stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, - bins, large, mutex, extents, hpa); - emitter_json_object_end( - emitter); /* Close "destroyed". */ - } - - /* Unmerged stats. */ - if (unmerged) { - for (i = 0; i < narenas; i++) { - if (initialized[i]) { - char arena_ind_str[20]; - malloc_snprintf(arena_ind_str, - sizeof(arena_ind_str), "%u", i); - emitter_json_object_kv_begin( - emitter, arena_ind_str); - emitter_table_printf(emitter, - "arenas[%s]:\n", arena_ind_str); - stats_arena_print(emitter, i, bins, - large, mutex, extents, hpa); - /* Close "". */ - emitter_json_object_end(emitter); - } - } - } - emitter_json_object_end(emitter); /* Close "stats.arenas". */ - } + g->num_background_threads, g->background_thread_num_runs, + g->background_thread_run_interval); } -void -stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) { - int err; - uint64_t epoch; - size_t u64sz; -#define OPTION(o, v, d, s) bool v = d; - STATS_PRINT_OPTIONS -#undef OPTION +static void +stats_global_mutexes_print(emitter_t *emitter) { + /* Counters are gathered into emitter columns; see the arena variant. */ + emitter_row_t row; + emitter_col_t name; + emitter_col_t col64[mutex_prof_num_uint64_t_counters]; + emitter_col_t col32[mutex_prof_num_uint32_t_counters]; + uint64_t uptime; - /* - * Refresh stats, in case mallctl() was called by the application. - * - * Check for OOM here, since refreshing the ctl cache can trigger - * allocation. In practice, none of the subsequent mallctl()-related - * calls in this function will cause OOM if this one succeeds. - * */ - epoch = 1; - u64sz = sizeof(uint64_t); - err = je_mallctl( - "epoch", (void *)&epoch, &u64sz, (void *)&epoch, sizeof(uint64_t)); - if (err != 0) { - if (err == EAGAIN) { - malloc_write( - ": Memory allocation failure in " - "mallctl(\"epoch\", ...)\n"); - return; - } - malloc_write( - ": Failure in mallctl(\"epoch\", " - "...)\n"); - abort(); + emitter_row_init(&row); + mutex_stats_init_cols(&row, "", &name, col64, col32); + + emitter_table_row(emitter, &row); + emitter_json_object_kv_begin(emitter, "mutexes"); + + CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); + + size_t stats_mutexes_mib[CTL_MAX_DEPTH]; + CTL_LEAF_PREPARE(stats_mutexes_mib, 0, "stats.mutexes"); + for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { + mutex_stats_read(stats_mutexes_mib, 2, + global_mutex_names[i], &name, col64, col32, uptime); + emitter_json_object_kv_begin(emitter, global_mutex_names[i]); + mutex_stats_emit(emitter, &row, col64, col32); + emitter_json_object_end(emitter); } - if (opts != NULL) { - for (unsigned i = 0; opts[i] != '\0'; i++) { - switch (opts[i]) { -#define OPTION(o, v, d, s) \ - case o: \ - v = s; \ - break; - STATS_PRINT_OPTIONS -#undef OPTION - default:; + emitter_json_object_end(emitter); /* Close "mutexes". */ +} + +static void +stats_print_globals(emitter_t *emitter, bool mutex) { + stats_global_t g; + stats_gather_global(&g); + + emitter_json_object_kv_begin(emitter, "stats"); + stats_emit_global(emitter, &g); + if (mutex) { + stats_global_mutexes_print(emitter); + } + emitter_json_object_end(emitter); /* Close "stats". */ +} + +static void +stats_print_one_arena(emitter_t *emitter, unsigned arena_ind, + const char *json_key, const char *table_header, bool bins, bool large, + bool mutex, bool extents, bool hpa) { + emitter_json_object_kv_begin(emitter, json_key); + emitter_table_printf(emitter, "%s", table_header); + stats_arena_print(emitter, arena_ind, bins, large, mutex, extents, hpa); + emitter_json_object_end(emitter); +} + +JEMALLOC_COLD +static void +stats_print_runtime_stats(emitter_t *emitter, bool merged, bool destroyed, + bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) { + stats_print_globals(emitter, mutex); + + if (!merged && !destroyed && !unmerged) { + return; + } + + unsigned narenas; + CTL_GET("arenas.narenas", &narenas, unsigned); + VARIABLE_ARRAY_UNSAFE(bool, initialized, narenas); + bool destroyed_initialized; + unsigned ninitialized = stats_gather_arenas_initialized(narenas, + initialized, &destroyed_initialized); + + emitter_json_object_kv_begin(emitter, "stats.arenas"); + + /* Merged stats. */ + if (merged && (ninitialized > 1 || !unmerged)) { + stats_print_one_arena(emitter, MALLCTL_ARENAS_ALL, "merged", + "Merged arenas stats:\n", bins, large, mutex, extents, hpa); + } + + /* Destroyed stats. */ + if (destroyed_initialized && destroyed) { + stats_print_one_arena(emitter, MALLCTL_ARENAS_DESTROYED, + "destroyed", "Destroyed arenas stats:\n", bins, large, mutex, + extents, hpa); + } + + /* Unmerged stats. */ + if (unmerged) { + for (unsigned i = 0; i < narenas; i++) { + if (!initialized[i]) { + continue; } + char arena_ind_str[20]; + malloc_snprintf(arena_ind_str, sizeof(arena_ind_str), + "%u", i); + char header[32]; + malloc_snprintf(header, sizeof(header), + "arenas[%s]:\n", arena_ind_str); + stats_print_one_arena(emitter, i, arena_ind_str, header, + bins, large, mutex, extents, hpa); } } - emitter_t emitter; - emitter_init(&emitter, - json ? emitter_output_json_compact : emitter_output_table, write_cb, - cbopaque); - emitter_begin(&emitter); - emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); - emitter_json_object_kv_begin(&emitter, "jemalloc"); - - if (general) { - stats_general_print(&emitter); - } - if (config_stats) { - stats_print_helper(&emitter, merged, destroyed, unmerged, bins, - large, mutex, extents, hpa); - } - - emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ - emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); - emitter_end(&emitter); + emitter_json_object_end(emitter); /* Close "stats.arenas". */ } static uint64_t diff --git a/test/unit/emitter.c b/test/unit/emitter.c index dc53b9eb..f209b773 100644 --- a/test/unit/emitter.c +++ b/test/unit/emitter.c @@ -498,13 +498,13 @@ emit_table_row(emitter_t *emitter) { emitter_begin(emitter); emitter_row_t row; emitter_col_t abc = { - emitter_justify_left, 10, emitter_type_title, {0}, {0, 0}}; + emitter_justify_left, 10, emitter_type_title}; abc.str_val = "ABC title"; emitter_col_t def = { - emitter_justify_right, 15, emitter_type_title, {0}, {0, 0}}; + emitter_justify_right, 15, emitter_type_title}; def.str_val = "DEF title"; emitter_col_t ghi = { - emitter_justify_right, 5, emitter_type_title, {0}, {0, 0}}; + emitter_justify_right, 5, emitter_type_title}; ghi.str_val = "GHI"; emitter_row_init(&row); @@ -548,6 +548,126 @@ static const char *table_row_table = "789 false 1011\n" "\"a string\" false ghi\n"; +/* + * emitter_row(): columns that carry a json_key render both to the aligned + * table (all columns) and, in JSON mode, to "json_key": value pairs in column + * order; columns without a json_key are table-only. + */ +static void +emit_row(emitter_t *emitter) { + emitter_begin(emitter); + + emitter_row_t row; + emitter_row_init(&row); + + /* Table-only column (no json_key set). */ + emitter_col_t col_label; + emitter_col_init(&col_label, &row); + col_label.justify = emitter_justify_left; + col_label.width = 6; + col_label.type = emitter_type_title; + col_label.str_val = "row:"; + + /* Dual columns: rendered in both table and JSON. */ + emitter_col_t col_x; + emitter_col_init(&col_x, &row); + col_x.justify = emitter_justify_right; + col_x.width = 6; + col_x.type = emitter_type_unsigned; + col_x.unsigned_val = 10; + col_x.json_key = "x"; + + emitter_col_t col_y; + emitter_col_init(&col_y, &row); + col_y.justify = emitter_justify_right; + col_y.width = 6; + col_y.type = emitter_type_unsigned; + col_y.unsigned_val = 20; + col_y.json_key = "y"; + + emitter_json_object_kv_begin(emitter, "obj"); + emitter_row(emitter, &row); + emitter_json_object_end(emitter); + + emitter_end(emitter); +} + +static const char *row_json = + "{\n" + "\t\"obj\": {\n" + "\t\t\"x\": 10,\n" + "\t\t\"y\": 20\n" + "\t}\n" + "}\n"; +static const char *row_json_compact = "{\"obj\":{\"x\":10,\"y\":20}}"; +static const char *row_table = "row: 10 20\n"; + +/* + * emitter_sparse_row(): the table side collapses runs of all-zero ("gap") rows + * to a single "---" separator (on a gap->non-gap transition and at a trailing + * gap), while the JSON side still emits every row's json-keyed columns. + */ +static void +emit_sparse_row(emitter_t *emitter) { + emitter_begin(emitter); + + emitter_row_t row; + emitter_row_init(&row); + emitter_col_t col_v; + emitter_col_init(&col_v, &row); + col_v.justify = emitter_justify_right; + col_v.width = 6; + col_v.type = emitter_type_unsigned; + col_v.json_key = "v"; + + unsigned vals[] = {1, 0, 0, 2, 0, 0}; + bool gap[] = {false, true, true, false, true, true}; + + emitter_json_array_kv_begin(emitter, "arr"); + emitter_table_sparse_begin(emitter); + for (unsigned i = 0; i < sizeof(vals) / sizeof(vals[0]); i++) { + col_v.unsigned_val = vals[i]; + emitter_json_object_begin(emitter); + emitter_sparse_row(emitter, &row, gap[i]); + emitter_json_object_end(emitter); + } + emitter_table_sparse_end(emitter); + emitter_json_array_end(emitter); + + emitter_end(emitter); +} + +static const char *sparse_row_json = + "{\n" + "\t\"arr\": [\n" + "\t\t{\n" + "\t\t\t\"v\": 1\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 2\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t},\n" + "\t\t{\n" + "\t\t\t\"v\": 0\n" + "\t\t}\n" + "\t]\n" + "}\n"; +static const char *sparse_row_json_compact = + "{\"arr\":[{\"v\":1},{\"v\":0},{\"v\":0},{\"v\":2},{\"v\":0},{\"v\":0}]}"; +static const char *sparse_row_table = + " 1\n" + " ---\n" + " 2\n" + " ---\n"; + #define GENERATE_TEST(feature) \ TEST_BEGIN(test_##feature) { \ expect_emit_output(emit_##feature, feature##_json, \ @@ -563,10 +683,12 @@ GENERATE_TEST(modal) GENERATE_TEST(json_array) GENERATE_TEST(json_nested_array) GENERATE_TEST(table_row) +GENERATE_TEST(row) +GENERATE_TEST(sparse_row) int main(void) { return test_no_reentrancy(test_dict, test_table_printf, test_nested_dict, test_types, test_modal, test_json_array, - test_json_nested_array, test_table_row); + test_json_nested_array, test_table_row, test_row, test_sparse_row); }