romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-18 14:47:18 +03:00
Code Issues Projects Releases Packages Wiki Activity
jemalloc/src/stats.c

2574 lines
93 KiB
C
Raw Blame History

#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/background_thread.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/emitter.h"
#include "jemalloc/internal/fxp.h"
#include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/prof_inlines.h"
#include "jemalloc/internal/prof_stats.h"
#include "jemalloc/internal/stats_internal.h"
#include "jemalloc/internal/tcache.h"
static const char *const global_mutex_names[mutex_prof_num_global_mutexes] = {
#define OP(mtx) #mtx,
MUTEX_PROF_GLOBAL_MUTEXES
#undef OP
};
static const char *const arena_mutex_names[mutex_prof_num_arena_mutexes] = {
#define OP(mtx) #mtx,
MUTEX_PROF_ARENA_MUTEXES
#undef OP
};
/******************************************************************************/
/* Data. */
bool opt_stats_print = false;
char opt_stats_print_opts[stats_print_tot_num_options + 1] = "";
int64_t opt_stats_interval = STATS_INTERVAL_DEFAULT;
char opt_stats_interval_opts[stats_print_tot_num_options + 1] = "";
static counter_accum_t stats_interval_accumulated;
/* Per thread batch accum size for stats_interval. */
uint64_t stats_interval_accum_batch;
/*
* Forward declarations for the two top-level section helpers. The public
* stats_print() appears first (just below) and calls these; their definitions
* follow it, so the file reads top-down.
*/
static void stats_general_print(emitter_t *emitter, bool omit_size_class_meta);
static void stats_print_runtime_stats(emitter_t *emitter, bool merged,
bool destroyed,
bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa);
/******************************************************************************/
/*
* malloc_stats_print() output structure (produced by stats_print(), below);
* each line names the function that prints it, plus its JSON key where useful.
* Quoted names are literal keys below the "jemalloc" root unless shown with a
* leading dot.
*
* ___ Begin jemalloc statistics ___
* { "jemalloc":
* general [g] -> stats_general_print
* version -> stats_general_version
* config -> stats_general_config
* system -> stats_general_system
* opt.* -> stats_general_opts
* prof -> stats_general_prof
* arenas metadata -> stats_general_arenas_print ("arenas")
* stats (config_stats) -> stats_print_runtime_stats
* global totals -> stats_print_globals ("stats")
* background_thread -> stats_print_globals
* mutexes [x] -> stats_global_mutexes_print
* per arena [m]/[d]/[a] -> stats_arena_print ("stats.arenas")
* basics -> stats_arena_basics_print
* decay -> stats_arena_decay_print
* alloc counts -> stats_arena_alloc_print (.small/.large)
* memory -> stats_arena_mem_print
* pac/sec -> stats_arena_pac_sec_print
* mutexes [x] -> stats_arena_mutexes_print (.mutexes)
* bins [b] -> stats_arena_bins_print (.bins[j])
* lextents [l] -> stats_arena_lextents_print (.lextents[j])
* extents [e] -> stats_arena_extents_print (.extents[j])
* hpa shard [h] -> stats_arena_hpa_shard_print (.hpa_shard)
* }
* --- End jemalloc statistics ---
*
* Most refactored sections read values through mallctl into typed structs
* before rendering them through the emitter. This makes their coverage easier
* to audit, but does not guarantee parity between formats: format-specific
* emission remains, including JSON-only geometry and table-only derived rates
* and summary rows.
*
* Mallctl names identify data sources, not mechanical JSON paths. For example,
* stats_gather_arena_alloc reads "stats.arenas.0.small.allocated" (replacing
* the MIB arena index at runtime), while the emitter writes it below
* jemalloc["stats.arenas"][<arena>].small.allocated. In particular, "stats"
* and "stats.arenas" are sibling keys, not nested objects.
*/
void
stats_print(write_cb_t *write_cb, void *cbopaque, const char *opts) {
int err;
uint64_t epoch;
size_t u64sz;
#define OPTION(o, v, d, s) bool v = d;
STATS_PRINT_OPTIONS
#undef OPTION
/*
* Refresh stats, in case mallctl() was called by the application.
*
* Check for OOM here, since refreshing the ctl cache can trigger
* allocation. In practice, none of the subsequent mallctl()-related
* calls in this function will cause OOM if this one succeeds.
* */
epoch = 1;
u64sz = sizeof(uint64_t);
err = je_mallctl(
"epoch", (void *)&epoch, &u64sz, (void *)&epoch, sizeof(uint64_t));
if (err != 0) {
if (err == EAGAIN) {
malloc_write(
"<jemalloc>: Memory allocation failure in "
"mallctl(\"epoch\", ...)\n");
return;
}
malloc_write(
"<jemalloc>: Failure in mallctl(\"epoch\", "
"...)\n");
abort();
}
if (opts != NULL) {
for (unsigned i = 0; opts[i] != '\0'; i++) {
switch (opts[i]) {
#define OPTION(o, v, d, s) \
case o: \
v = s; \
break;
STATS_PRINT_OPTIONS
#undef OPTION
default:;
}
}
}
emitter_t emitter;
emitter_init(&emitter,
json ? emitter_output_json_compact : emitter_output_table, write_cb,
cbopaque);
emitter_begin(&emitter);
emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n");
emitter_json_object_kv_begin(&emitter, "jemalloc");
if (general) {
/*
* Private (non-public) omission: the per-size-class geometry
* arrays are shown only in JSON; the raw format never did.
*/
bool omit_size_class_meta = !json;
stats_general_print(&emitter, omit_size_class_meta);
}
if (config_stats) {
stats_print_runtime_stats(&emitter, merged, destroyed, unmerged,
bins, large, mutex, extents, hpa);
}
emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */
emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n");
emitter_end(&emitter);
}
/******************************************************************************/
/* Statistics gathering. */
/*
* Read stats out of the allocator through mallctl into typed per-section
* structs. Keeping the mallctl reads together (and separate from emission)
* makes coverage auditable. They are static to src/stats.c; the gather structs
* and plumbing macros they use live in stats_internal.h.
*/
/* mib must be prepared through "stats.arenas.<i>.extents" (depth 4). */
static void
stats_gather_arena_extent(size_t *mib, unsigned j, stats_arena_extent_t *e) {
mib[4] = j;
CTL_LEAF(mib, 5, "ndirty", &e->ndirty, size_t);
CTL_LEAF(mib, 5, "nmuzzy", &e->nmuzzy, size_t);
CTL_LEAF(mib, 5, "nretained", &e->nretained, size_t);
CTL_LEAF(mib, 5, "npinned", &e->npinned, size_t);
CTL_LEAF(mib, 5, "dirty_bytes", &e->dirty_bytes, size_t);
CTL_LEAF(mib, 5, "muzzy_bytes", &e->muzzy_bytes, size_t);
CTL_LEAF(mib, 5, "retained_bytes", &e->retained_bytes, size_t);
CTL_LEAF(mib, 5, "pinned_bytes", &e->pinned_bytes, size_t);
e->ntotal = e->ndirty + e->nmuzzy + e->nretained + e->npinned;
e->total_bytes = e->dirty_bytes + e->muzzy_bytes + e->retained_bytes
+ e->pinned_bytes;
}
static void
stats_gather_arena_mem(unsigned i, stats_arena_mem_t *mem) {
CTL_M2_GET("stats.arenas.0.mapped", i, &mem->mapped, size_t);
CTL_M2_GET("stats.arenas.0.retained", i, &mem->retained, size_t);
CTL_M2_GET("stats.arenas.0.pinned", i, &mem->pinned, size_t);
CTL_M2_GET("stats.arenas.0.base", i, &mem->base, size_t);
CTL_M2_GET("stats.arenas.0.internal", i, &mem->internal, size_t);
CTL_M2_GET("stats.arenas.0.metadata_edata", i, &mem->metadata_edata,
size_t);
CTL_M2_GET("stats.arenas.0.metadata_rtree", i, &mem->metadata_rtree,
size_t);
CTL_M2_GET("stats.arenas.0.metadata_thp", i, &mem->metadata_thp,
size_t);
CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &mem->tcache_bytes,
size_t);
CTL_M2_GET("stats.arenas.0.tcache_stashed_bytes", i,
&mem->tcache_stashed_bytes, size_t);
CTL_M2_GET("stats.arenas.0.resident", i, &mem->resident, size_t);
CTL_M2_GET("stats.arenas.0.abandoned_vm", i, &mem->abandoned_vm,
size_t);
CTL_M2_GET("stats.arenas.0.extent_avail", i, &mem->extent_avail,
size_t);
CTL_GET("arenas.page", &mem->page, size_t);
}
static void
stats_gather_arena_basics(unsigned i, stats_arena_basics_t *b) {
CTL_M2_GET("stats.arenas.0.nthreads", i, &b->nthreads, unsigned);
CTL_M2_GET("stats.arenas.0.uptime", i, &b->uptime, uint64_t);
CTL_M2_GET("stats.arenas.0.dss", i, &b->dss, const char *);
/*
* The arena.<i>.name ctl copies the name into the caller-provided
* buffer (b->name), so we hand it a real buffer rather than reading
* into a borrowed pointer. The merged/destroyed pseudo-arenas have no
* name, and querying them would fail, so guard the read.
*/
b->has_name = (i != MALLCTL_ARENAS_ALL && i != MALLCTL_ARENAS_DESTROYED);
if (b->has_name) {
char *namep = b->name;
CTL_M1_GET("arena.0.name", i, (void *)&namep, const char *);
}
}
static void
stats_gather_arena_decay(unsigned i, stats_arena_decay_t *d) {
CTL_M2_GET(
"stats.arenas.0.dirty_decay_ms", i, &d->dirty_decay_ms, ssize_t);
CTL_M2_GET(
"stats.arenas.0.muzzy_decay_ms", i, &d->muzzy_decay_ms, ssize_t);
CTL_M2_GET("stats.arenas.0.pactive", i, &d->pactive, size_t);
CTL_M2_GET("stats.arenas.0.pdirty", i, &d->pdirty, size_t);
CTL_M2_GET("stats.arenas.0.pmuzzy", i, &d->pmuzzy, size_t);
CTL_M2_GET("stats.arenas.0.dirty_npurge", i, &d->dirty_npurge, uint64_t);
CTL_M2_GET(
"stats.arenas.0.dirty_nmadvise", i, &d->dirty_nmadvise, uint64_t);
CTL_M2_GET("stats.arenas.0.dirty_purged", i, &d->dirty_purged, uint64_t);
CTL_M2_GET("stats.arenas.0.muzzy_npurge", i, &d->muzzy_npurge, uint64_t);
CTL_M2_GET(
"stats.arenas.0.muzzy_nmadvise", i, &d->muzzy_nmadvise, uint64_t);
CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &d->muzzy_purged, uint64_t);
}
static void
stats_gather_arena_alloc(unsigned i, stats_arena_alloc_t *small,
stats_arena_alloc_t *large) {
CTL_M2_GET(
"stats.arenas.0.small.allocated", i, &small->allocated, size_t);
CTL_M2_GET("stats.arenas.0.small.nmalloc", i, &small->nmalloc, uint64_t);
CTL_M2_GET("stats.arenas.0.small.ndalloc", i, &small->ndalloc, uint64_t);
CTL_M2_GET(
"stats.arenas.0.small.nrequests", i, &small->nrequests, uint64_t);
CTL_M2_GET("stats.arenas.0.small.nfills", i, &small->nfills, uint64_t);
CTL_M2_GET("stats.arenas.0.small.nflushes", i, &small->nflushes,
uint64_t);
CTL_M2_GET(
"stats.arenas.0.large.allocated", i, &large->allocated, size_t);
CTL_M2_GET("stats.arenas.0.large.nmalloc", i, &large->nmalloc, uint64_t);
CTL_M2_GET("stats.arenas.0.large.ndalloc", i, &large->ndalloc, uint64_t);
CTL_M2_GET(
"stats.arenas.0.large.nrequests", i, &large->nrequests, uint64_t);
CTL_M2_GET("stats.arenas.0.large.nfills", i, &large->nfills, uint64_t);
CTL_M2_GET("stats.arenas.0.large.nflushes", i, &large->nflushes,
uint64_t);
}
/*
* nslabs + prof (which gate the table gap/skip) are read by the caller; this
* reads the remaining detail fields. mibs prepared + indexed by the caller.
*/
static void
stats_gather_arena_bin(size_t *stats_mib, size_t *arenas_bin_mib,
stats_arena_bin_t *bin) {
CTL_LEAF(arenas_bin_mib, 3, "size", &bin->reg_size, size_t);
CTL_LEAF(arenas_bin_mib, 3, "nregs", &bin->nregs, uint32_t);
CTL_LEAF(arenas_bin_mib, 3, "slab_size", &bin->slab_size, size_t);
CTL_LEAF(arenas_bin_mib, 3, "nshards", &bin->nshards, uint32_t);
CTL_LEAF(stats_mib, 5, "nmalloc", &bin->nmalloc, uint64_t);
CTL_LEAF(stats_mib, 5, "ndalloc", &bin->ndalloc, uint64_t);
CTL_LEAF(stats_mib, 5, "curregs", &bin->curregs, size_t);
CTL_LEAF(stats_mib, 5, "nrequests", &bin->nrequests, uint64_t);
CTL_LEAF(stats_mib, 5, "nfills", &bin->nfills, uint64_t);
CTL_LEAF(stats_mib, 5, "nflushes", &bin->nflushes, uint64_t);
CTL_LEAF(stats_mib, 5, "nreslabs", &bin->nreslabs, uint64_t);
CTL_LEAF(stats_mib, 5, "curslabs", &bin->curslabs, size_t);
CTL_LEAF(stats_mib, 5, "nonfull_slabs", &bin->nonfull_slabs, size_t);
}
static void
stats_gather_arena_lextent(size_t *stats_mib, size_t *arenas_lextent_mib,
size_t *prof_stats_mib, unsigned j, bool prof_stats_on,
stats_arena_lextent_t *lext) {
stats_mib[4] = j;
arenas_lextent_mib[2] = j;
CTL_LEAF(stats_mib, 5, "nmalloc", &lext->nmalloc, uint64_t);
CTL_LEAF(stats_mib, 5, "ndalloc", &lext->ndalloc, uint64_t);
CTL_LEAF(stats_mib, 5, "nrequests", &lext->nrequests, uint64_t);
CTL_LEAF(arenas_lextent_mib, 3, "size", &lext->lextent_size, size_t);
CTL_LEAF(stats_mib, 5, "curlextents", &lext->curlextents, size_t);
if (prof_stats_on) {
prof_stats_mib[3] = j;
CTL_LEAF(prof_stats_mib, 4, "live", &lext->prof_live,
prof_stats_t);
CTL_LEAF(prof_stats_mib, 4, "accum", &lext->prof_accum,
prof_stats_t);
}
}
static void
stats_gather_arena_hpa_sec(unsigned i, stats_arena_hpa_sec_t *sec) {
CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec->sec_bytes, size_t);
CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec->sec_hits, size_t);
CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec->sec_misses, size_t);
CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i,
&sec->sec_dalloc_noflush, size_t);
CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i,
&sec->sec_dalloc_flush, size_t);
CTL_M2_GET("stats.arenas.0.hpa_sec_overfills", i, &sec->sec_overfills,
size_t);
}
static void
stats_gather_arena_pac_sec(unsigned i, stats_arena_pac_sec_t *sec) {
CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec->sec_bytes, size_t);
CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec->sec_hits, size_t);
CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec->sec_misses, size_t);
CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i,
&sec->sec_dalloc_noflush, size_t);
CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i,
&sec->sec_dalloc_flush, size_t);
}
static void
stats_gather_arena_hpa_counters(unsigned i, stats_arena_hpa_counters_t *c) {
CTL_M2_GET(
"stats.arenas.0.hpa_shard.npageslabs", i, &c->npageslabs, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.nactive", i, &c->nactive, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.ndirty", i, &c->ndirty, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_nonhuge", i,
&c->npageslabs_nonhuge, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_nonhuge", i,
&c->nactive_nonhuge, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_nonhuge", i,
&c->ndirty_nonhuge, size_t);
c->nretained_nonhuge = c->npageslabs_nonhuge * HUGEPAGE_PAGES
- c->nactive_nonhuge - c->ndirty_nonhuge;
CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.npageslabs_huge", i,
&c->npageslabs_huge, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.nactive_huge", i,
&c->nactive_huge, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.slabs.ndirty_huge", i,
&c->ndirty_huge, size_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.npurge_passes", i,
&c->npurge_passes, uint64_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.npurges", i, &c->npurges, uint64_t);
CTL_M2_GET(
"stats.arenas.0.hpa_shard.nhugifies", i, &c->nhugifies, uint64_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.nhugify_failures", i,
&c->nhugify_failures, uint64_t);
CTL_M2_GET("stats.arenas.0.hpa_shard.ndehugifies", i, &c->ndehugifies,
uint64_t);
}
/* kind is "full_slabs" or "empty_slabs". */
static void
stats_gather_arena_hpa_slab(unsigned i, const char *kind,
stats_arena_hpa_slab_t *s) {
size_t mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(mib, 0, "stats.arenas");
mib[2] = i;
CTL_LEAF_PREPARE(mib, 3, "hpa_shard");
CTL_LEAF_PREPARE(mib, 4, kind);
CTL_LEAF(mib, 5, "npageslabs_huge", &s->npageslabs_huge, size_t);
CTL_LEAF(mib, 5, "nactive_huge", &s->nactive_huge, size_t);
CTL_LEAF(mib, 5, "ndirty_huge", &s->ndirty_huge, size_t);
CTL_LEAF(mib, 5, "npageslabs_nonhuge", &s->npageslabs_nonhuge, size_t);
CTL_LEAF(mib, 5, "nactive_nonhuge", &s->nactive_nonhuge, size_t);
CTL_LEAF(mib, 5, "ndirty_nonhuge", &s->ndirty_nonhuge, size_t);
s->nretained_nonhuge = s->npageslabs_nonhuge * HUGEPAGE_PAGES
- s->nactive_nonhuge - s->ndirty_nonhuge;
}
/* mib prepared through "stats.arenas.<i>.hpa_shard.nonfull_slabs" (depth 5). */
static void
stats_gather_arena_hpa_nonfull(size_t *mib, unsigned j,
stats_arena_hpa_slab_t *s) {
mib[5] = j;
CTL_LEAF(mib, 6, "npageslabs_huge", &s->npageslabs_huge, size_t);
CTL_LEAF(mib, 6, "nactive_huge", &s->nactive_huge, size_t);
CTL_LEAF(mib, 6, "ndirty_huge", &s->ndirty_huge, size_t);
CTL_LEAF(mib, 6, "npageslabs_nonhuge", &s->npageslabs_nonhuge, size_t);
CTL_LEAF(mib, 6, "nactive_nonhuge", &s->nactive_nonhuge, size_t);
CTL_LEAF(mib, 6, "ndirty_nonhuge", &s->ndirty_nonhuge, size_t);
s->nretained_nonhuge = s->npageslabs_nonhuge * HUGEPAGE_PAGES
- s->nactive_nonhuge - s->ndirty_nonhuge;
}
static void
stats_gather_arena_config(stats_arena_config_t *cfg) {
CTL_GET("arenas.narenas", &cfg->narenas, unsigned);
CTL_GET("arenas.dirty_decay_ms", &cfg->dirty_decay_ms, ssize_t);
CTL_GET("arenas.muzzy_decay_ms", &cfg->muzzy_decay_ms, ssize_t);
CTL_GET("arenas.quantum", &cfg->quantum, size_t);
CTL_GET("arenas.page", &cfg->page, size_t);
CTL_GET("arenas.hugepage", &cfg->hugepage, size_t);
size_t sz = sizeof(size_t);
cfg->have_tcache_max = (je_mallctl("arenas.tcache_max",
(void *)&cfg->tcache_max, &sz, NULL, 0) == 0);
CTL_GET("arenas.nbins", &cfg->nbins, unsigned);
CTL_GET("arenas.nhbins", &cfg->nhbins, unsigned);
CTL_GET("arenas.nlextents", &cfg->nlextents, unsigned);
}
/* mib must be prepared through "arenas.bin" (depth 2). */
static void
stats_gather_arena_bin_meta(size_t *mib, unsigned j,
stats_arena_bin_meta_t *bm) {
mib[2] = j;
CTL_LEAF(mib, 3, "size", &bm->size, size_t);
CTL_LEAF(mib, 3, "nregs", &bm->nregs, uint32_t);
CTL_LEAF(mib, 3, "slab_size", &bm->slab_size, size_t);
CTL_LEAF(mib, 3, "nshards", &bm->nshards, uint32_t);
}
/* mib must be prepared through "arenas.lextent" (depth 2). */
static void
stats_gather_arena_lextent_meta(size_t *mib, unsigned j,
stats_arena_lextent_meta_t *lm) {
mib[2] = j;
CTL_LEAF(mib, 3, "size", &lm->size, size_t);
}
static void
stats_gather_global(stats_global_t *g) {
CTL_GET("stats.allocated", &g->allocated, size_t);
CTL_GET("stats.active", &g->active, size_t);
CTL_GET("stats.metadata", &g->metadata, size_t);
CTL_GET("stats.metadata_edata", &g->metadata_edata, size_t);
CTL_GET("stats.metadata_rtree", &g->metadata_rtree, size_t);
CTL_GET("stats.metadata_thp", &g->metadata_thp, size_t);
CTL_GET("stats.resident", &g->resident, size_t);
CTL_GET("stats.mapped", &g->mapped, size_t);
CTL_GET("stats.retained", &g->retained, size_t);
CTL_GET("stats.pinned", &g->pinned, size_t);
CTL_GET("stats.zero_reallocs", &g->zero_reallocs, size_t);
if (have_background_thread) {
CTL_GET("stats.background_thread.num_threads",
&g->num_background_threads, size_t);
CTL_GET("stats.background_thread.num_runs",
&g->background_thread_num_runs, uint64_t);
CTL_GET("stats.background_thread.run_interval",
&g->background_thread_run_interval, uint64_t);
} else {
g->num_background_threads = 0;
g->background_thread_num_runs = 0;
g->background_thread_run_interval = 0;
}
}
/*
* Fills initialized[narenas] and *destroyed_initialized; returns the number of
* initialized arenas.
*/
static unsigned
stats_gather_arenas_initialized(unsigned narenas, bool *initialized,
bool *destroyed_initialized) {
size_t mib[3];
size_t miblen = sizeof(mib) / sizeof(size_t);
size_t sz;
unsigned ninitialized = 0;
xmallctlnametomib("arena.0.initialized", mib, &miblen);
for (unsigned i = 0; i < narenas; i++) {
mib[1] = i;
sz = sizeof(bool);
xmallctlbymib(mib, miblen, &initialized[i], &sz, NULL, 0);
if (initialized[i]) {
ninitialized++;
}
}
mib[1] = MALLCTL_ARENAS_DESTROYED;
sz = sizeof(bool);
xmallctlbymib(mib, miblen, destroyed_initialized, &sz, NULL, 0);
return ninitialized;
}
/******************************************************************************/
static uint64_t
rate_per_second(uint64_t value, uint64_t uptime_ns) {
uint64_t billion = 1000000000;
if (uptime_ns == 0 || value == 0) {
return 0;
}
if (uptime_ns < billion) {
return value;
} else {
uint64_t uptime_s = uptime_ns / billion;
return value / uptime_s;
}
}
/* Calculate x.yyy and output a string (takes a fixed sized char array). */
static bool
get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) {
if (divisor == 0 || dividend > divisor) {
/* The rate is not supposed to be greater than 1. */
return true;
}
if (dividend > 0) {
assert(UINT64_MAX / dividend >= 1000);
}
unsigned n = (unsigned)((dividend * 1000) / divisor);
if (n < 10) {
malloc_snprintf(str, 6, "0.00%u", n);
} else if (n < 100) {
malloc_snprintf(str, 6, "0.0%u", n);
} else if (n < 1000) {
malloc_snprintf(str, 6, "0.%u", n);
} else {
malloc_snprintf(str, 6, "1");
}
return false;
}
static void
mutex_stats_init_cols(emitter_row_t *row, const char *table_name,
emitter_col_t *name,
emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) {
mutex_prof_uint64_t_counter_ind_t k_uint64_t = 0;
mutex_prof_uint32_t_counter_ind_t k_uint32_t = 0;
emitter_col_t *col;
if (name != NULL) {
emitter_col_init(name, row);
name->justify = emitter_justify_left;
name->width = 21;
name->type = emitter_type_title;
name->str_val = table_name;
}
#define WIDTH_uint32_t 12
#define WIDTH_uint64_t 16
#define OP(counter, counter_type, human, derived, base_counter) \
col = &col_##counter_type[k_##counter_type]; \
++k_##counter_type; \
emitter_col_init(col, row); \
col->justify = emitter_justify_right; \
col->width = derived ? 8 : WIDTH_##counter_type; \
col->type = emitter_type_title; \
col->str_val = human;
MUTEX_PROF_COUNTERS
#undef OP
#undef WIDTH_uint32_t
#undef WIDTH_uint64_t
col_uint64_t[mutex_counter_total_wait_time_ps].width = 10;
}
/*
* Read one mutex's counters (leaf "name" under the prepared mib) into the
* emitter columns. Shared by the global and per-arena mutex tables, whose rows
* differ only in the mib and mutex name they pass.
*/
static void
mutex_stats_read(size_t mib[], size_t miblen, const char *name,
emitter_col_t *col_name,
emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
uint64_t uptime) {
CTL_LEAF_PREPARE(mib, miblen, name);
size_t miblen_name = miblen + 1;
col_name->str_val = name;
emitter_col_t *dst;
#define EMITTER_TYPE_uint32_t emitter_type_uint32
#define EMITTER_TYPE_uint64_t emitter_type_uint64
#define OP(counter, counter_type, human, derived, base_counter) \
dst = &col_##counter_type[mutex_counter_##counter]; \
dst->type = EMITTER_TYPE_##counter_type; \
if (!derived) { \
CTL_LEAF(mib, miblen_name, #counter, \
(counter_type *)&dst->bool_val, counter_type); \
} else { \
emitter_col_t *base = \
&col_##counter_type[mutex_counter_##base_counter]; \
dst->counter_type##_val = (counter_type)rate_per_second( \
base->counter_type##_val, uptime); \
}
MUTEX_PROF_COUNTERS
#undef OP
#undef EMITTER_TYPE_uint32_t
#undef EMITTER_TYPE_uint64_t
}
static void
mutex_stats_read_arena_bin(size_t mib[], size_t miblen,
emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters],
uint64_t uptime) {
CTL_LEAF_PREPARE(mib, miblen, "mutex");
size_t miblen_mutex = miblen + 1;
emitter_col_t *dst;
#define EMITTER_TYPE_uint32_t emitter_type_uint32
#define EMITTER_TYPE_uint64_t emitter_type_uint64
#define OP(counter, counter_type, human, derived, base_counter) \
dst = &col_##counter_type[mutex_counter_##counter]; \
dst->type = EMITTER_TYPE_##counter_type; \
if (!derived) { \
CTL_LEAF(mib, miblen_mutex, #counter, \
(counter_type *)&dst->bool_val, counter_type); \
} else { \
emitter_col_t *base = \
&col_##counter_type[mutex_counter_##base_counter]; \
dst->counter_type##_val = (counter_type)rate_per_second( \
base->counter_type##_val, uptime); \
}
MUTEX_PROF_COUNTERS
#undef OP
#undef EMITTER_TYPE_uint32_t
#undef EMITTER_TYPE_uint64_t
}
/* "row" can be NULL to avoid emitting in table mode. */
static void
mutex_stats_emit(emitter_t *emitter, emitter_row_t *row,
emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters],
emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) {
if (row != NULL) {
emitter_table_row(emitter, row);
}
mutex_prof_uint64_t_counter_ind_t k_uint64_t = 0;
mutex_prof_uint32_t_counter_ind_t k_uint32_t = 0;
emitter_col_t *col;
#define EMITTER_TYPE_uint32_t emitter_type_uint32
#define EMITTER_TYPE_uint64_t emitter_type_uint64
#define OP(counter, type, human, derived, base_counter) \
if (!derived) { \
col = &col_##type[k_##type]; \
emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \
(const void *)&col->bool_val); \
} \
++k_##type;
MUTEX_PROF_COUNTERS;
#undef OP
#undef EMITTER_TYPE_uint32_t
#undef EMITTER_TYPE_uint64_t
}
/*
* Set a per-size-class row's "size" column. Above the slow-growth threshold,
* display the bucket bounds as "(prev_size,size]". This notation does not
* imply that every integer in the interval is a valid usable size. Smaller
* classes are displayed as a single size. buf must outlive row emission.
* (This is table labeling only; the ordered JSON arrays imply the bounds.)
*/
static void
stats_size_col_set(emitter_col_t *col, size_t size, size_t prev_size, char *buf,
size_t bufsz) {
if (size > USIZE_GROW_SLOW_THRESHOLD) {
malloc_snprintf(buf, bufsz, "(%zu,%zu]", prev_size, size);
col->type = emitter_type_title;
col->str_val = buf;
} else {
col->type = emitter_type_size;
col->size_val = size;
}
}
enum {
STATS_COL_FLAG_NONE = 0,
STATS_COL_FLAG_PROF = 1U << 0,
};
static unsigned
stats_col_active_flags(bool prof_stats_on) {
return prof_stats_on ? STATS_COL_FLAG_PROF : STATS_COL_FLAG_NONE;
}
typedef struct {
const stats_arena_bin_t *bin;
unsigned ind;
size_t page;
uint64_t uptime;
const char *util;
} stats_arena_bin_emit_row_t;
#define BIN_COL_GET(name, value_member, field) \
static void \
stats_bin_col_get_##name(const void *vrow, emitter_col_t *col) { \
const stats_arena_bin_emit_row_t *row = vrow; \
col->value_member = row->bin->field; \
}
#define BIN_COL_GET_RATE(name, field) \
static void \
stats_bin_col_get_##name(const void *vrow, emitter_col_t *col) { \
const stats_arena_bin_emit_row_t *row = vrow; \
col->uint64_val = rate_per_second( \
row->bin->field, row->uptime); \
}
BIN_COL_GET(size, size_val, reg_size)
BIN_COL_GET(nmalloc, uint64_val, nmalloc)
BIN_COL_GET(ndalloc, uint64_val, ndalloc)
BIN_COL_GET(nrequests, uint64_val, nrequests)
BIN_COL_GET(prof_live_requested, uint64_val, prof_live.req_sum)
BIN_COL_GET(prof_live_count, uint64_val, prof_live.count)
BIN_COL_GET(prof_accum_requested, uint64_val, prof_accum.req_sum)
BIN_COL_GET(prof_accum_count, uint64_val, prof_accum.count)
BIN_COL_GET(nshards, unsigned_val, nshards)
BIN_COL_GET(curregs, size_val, curregs)
BIN_COL_GET(curslabs, size_val, curslabs)
BIN_COL_GET(nonfull_slabs, size_val, nonfull_slabs)
BIN_COL_GET(regs, unsigned_val, nregs)
BIN_COL_GET(nfills, uint64_val, nfills)
BIN_COL_GET(nflushes, uint64_val, nflushes)
BIN_COL_GET(nslabs, uint64_val, nslabs)
BIN_COL_GET(nreslabs, uint64_val, nreslabs)
BIN_COL_GET_RATE(nmalloc_ps, nmalloc)
BIN_COL_GET_RATE(ndalloc_ps, ndalloc)
BIN_COL_GET_RATE(nrequests_ps, nrequests)
BIN_COL_GET_RATE(nfills_ps, nfills)
BIN_COL_GET_RATE(nflushes_ps, nflushes)
BIN_COL_GET_RATE(nreslabs_ps, nreslabs)
#undef BIN_COL_GET
#undef BIN_COL_GET_RATE
static void
stats_bin_col_get_ind(const void *vrow, emitter_col_t *col) {
const stats_arena_bin_emit_row_t *row = vrow;
col->unsigned_val = row->ind;
}
static void
stats_bin_col_get_allocated(const void *vrow, emitter_col_t *col) {
const stats_arena_bin_emit_row_t *row = vrow;
col->size_val = row->bin->curregs * row->bin->reg_size;
}
static void
stats_bin_col_get_pgs(const void *vrow, emitter_col_t *col) {
const stats_arena_bin_emit_row_t *row = vrow;
col->size_val = row->bin->slab_size / row->page;
}
static void
stats_bin_col_get_spacer(const void *vrow, emitter_col_t *col) {
(void)vrow;
col->str_val = " ";
}
static void
stats_bin_col_get_util(const void *vrow, emitter_col_t *col) {
const stats_arena_bin_emit_row_t *row = vrow;
col->str_val = row->util;
}
#define BIN_COL_SIZE 0
#define BIN_DESC(key, label, width, type, flags, name) \
{key, label, emitter_justify_right, width, emitter_type_##type, flags, \
stats_bin_col_get_##name}
static const emitter_col_desc_t stats_bin_cols[] = {
BIN_DESC("size", "size", 20, size, STATS_COL_FLAG_NONE, size),
BIN_DESC("ind", "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind),
BIN_DESC("allocated", "allocated", 14, size, STATS_COL_FLAG_NONE,
allocated),
BIN_DESC("nmalloc", "nmalloc", 14, uint64, STATS_COL_FLAG_NONE,
nmalloc),
BIN_DESC("nmalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE,
nmalloc_ps),
BIN_DESC("ndalloc", "ndalloc", 14, uint64, STATS_COL_FLAG_NONE,
ndalloc),
BIN_DESC("ndalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE,
ndalloc_ps),
BIN_DESC("nrequests", "nrequests", 15, uint64, STATS_COL_FLAG_NONE,
nrequests),
BIN_DESC("nrequests_ps", "(#/sec)", 10, uint64, STATS_COL_FLAG_NONE,
nrequests_ps),
BIN_DESC("prof_live_requested", "prof_live_requested", 21, uint64,
STATS_COL_FLAG_PROF, prof_live_requested),
BIN_DESC("prof_live_count", "prof_live_count", 17, uint64,
STATS_COL_FLAG_PROF, prof_live_count),
BIN_DESC("prof_accum_requested", "prof_accum_requested", 21, uint64,
STATS_COL_FLAG_PROF, prof_accum_requested),
BIN_DESC("prof_accum_count", "prof_accum_count", 17, uint64,
STATS_COL_FLAG_PROF, prof_accum_count),
BIN_DESC("nshards", "nshards", 9, unsigned, STATS_COL_FLAG_NONE,
nshards),
BIN_DESC("curregs", "curregs", 13, size, STATS_COL_FLAG_NONE, curregs),
BIN_DESC("curslabs", "curslabs", 13, size, STATS_COL_FLAG_NONE,
curslabs),
BIN_DESC("nonfull_slabs", "nonfull_slabs", 15, size,
STATS_COL_FLAG_NONE,
nonfull_slabs),
BIN_DESC("regs", "regs", 5, unsigned, STATS_COL_FLAG_NONE, regs),
BIN_DESC("pgs", "pgs", 4, size, STATS_COL_FLAG_NONE, pgs),
BIN_DESC(NULL, " ", 1, title, STATS_COL_FLAG_NONE, spacer),
BIN_DESC("util", "util", 6, title, STATS_COL_FLAG_NONE, util),
BIN_DESC("nfills", "nfills", 13, uint64, STATS_COL_FLAG_NONE, nfills),
BIN_DESC("nfills_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE,
nfills_ps),
BIN_DESC("nflushes", "nflushes", 13, uint64, STATS_COL_FLAG_NONE,
nflushes),
BIN_DESC("nflushes_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE,
nflushes_ps),
BIN_DESC("nslabs", "nslabs", 13, uint64, STATS_COL_FLAG_NONE, nslabs),
BIN_DESC("nreslabs", "nreslabs", 13, uint64, STATS_COL_FLAG_NONE,
nreslabs),
BIN_DESC("nreslabs_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE,
nreslabs_ps),
};
#undef BIN_DESC
#define BIN_COL_COUNT (sizeof(stats_bin_cols) / sizeof(stats_bin_cols[0]))
static void
stats_emit_arena_bin_row(emitter_t *emitter, emitter_row_t *table_row,
emitter_col_t *cols, const stats_arena_bin_emit_row_t *row,
bool prof_stats_on, bool mutex, emitter_col_t *mutex64,
emitter_col_t *mutex32, bool is_gap) {
unsigned active_flags = stats_col_active_flags(prof_stats_on);
emitter_col_table_fill(
stats_bin_cols, BIN_COL_COUNT, active_flags, cols, row);
emitter_json_object_begin(emitter);
emitter_col_table_emit_json(emitter, stats_bin_cols, BIN_COL_COUNT,
active_flags, cols);
if (mutex) {
emitter_json_object_kv_begin(emitter, "mutex");
mutex_stats_emit(emitter, NULL, mutex64, mutex32);
emitter_json_object_end(emitter);
}
emitter_json_object_end(emitter);
emitter_table_sparse_row(emitter, table_row, is_gap);
}
JEMALLOC_COLD
static void
stats_arena_bins_print(
emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) {
/*
* Process one size-class row at a time to preserve O(1) memory. Full
* gather-all-rows-before-emit separation would require buffering the
* table, so gathering and emission remain interleaved across rows. The
* lextents, extents, and HPA nonfull tables follow the same pattern.
*/
size_t page;
unsigned nbins, j;
CTL_GET("arenas.page", &page, size_t);
CTL_GET("arenas.nbins", &nbins, unsigned);
bool prof_stats_on = config_prof && opt_prof && opt_prof_stats
&& i == MALLCTL_ARENAS_ALL;
emitter_row_t row, header_row;
emitter_col_t cols[BIN_COL_COUNT], header_cols[BIN_COL_COUNT];
emitter_col_table_build(stats_bin_cols, BIN_COL_COUNT,
stats_col_active_flags(prof_stats_on), &row, cols, &header_row,
header_cols);
emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters];
emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters];
emitter_col_t header_mutex64[mutex_prof_num_uint64_t_counters];
emitter_col_t header_mutex32[mutex_prof_num_uint32_t_counters];
if (mutex) {
mutex_stats_init_cols(
&row, NULL, NULL, col_mutex64, col_mutex32);
mutex_stats_init_cols(
&header_row, NULL, NULL, header_mutex64, header_mutex32);
}
emitter_col_table_header(emitter, &header_row,
&header_cols[BIN_COL_SIZE], "bins:", "bins");
size_t stats_arenas_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
stats_arenas_mib[2] = i;
CTL_LEAF_PREPARE(stats_arenas_mib, 3, "bins");
size_t arenas_bin_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(arenas_bin_mib, 0, "arenas.bin");
size_t prof_stats_mib[CTL_MAX_DEPTH];
if (prof_stats_on) {
CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.bins");
}
emitter_table_sparse_begin(emitter);
for (j = 0; j < nbins; j++) {
stats_arena_bin_t bin;
stats_arenas_mib[4] = j;
arenas_bin_mib[2] = j;
CTL_LEAF(stats_arenas_mib, 5, "nslabs", &bin.nslabs, uint64_t);
if (prof_stats_on) {
prof_stats_mib[3] = j;
CTL_LEAF(prof_stats_mib, 4, "live", &bin.prof_live,
prof_stats_t);
CTL_LEAF(prof_stats_mib, 4, "accum", &bin.prof_accum,
prof_stats_t);
}
bool is_gap;
if (prof_stats_on) {
is_gap = (bin.nslabs == 0 && bin.prof_accum.count == 0);
} else {
is_gap = (bin.nslabs == 0);
}
if (is_gap && !emitter_outputs_json(emitter)) {
emitter_table_sparse_row(emitter, &row, true);
continue;
}
/* Mutex counters still gather directly into their emitter columns. */
stats_gather_arena_bin(stats_arenas_mib, arenas_bin_mib, &bin);
if (mutex) {
mutex_stats_read_arena_bin(stats_arenas_mib, 5,
col_mutex64, col_mutex32, uptime);
}
size_t availregs = bin.nregs * bin.curslabs;
char util[6];
if (get_rate_str(
(uint64_t)bin.curregs, (uint64_t)availregs, util)) {
if (availregs == 0) {
malloc_snprintf(util, sizeof(util), "1");
} else if (bin.curregs > availregs) {
/*
* Race detected: the counters were read in
* separate mallctl calls and concurrent
* operations happened in between. In this case
* no meaningful utilization can be computed.
*/
malloc_snprintf(util, sizeof(util), " race");
} else {
not_reached();
}
}
stats_arena_bin_emit_row_t emit_row = {
&bin, j, page, uptime, util};
stats_emit_arena_bin_row(emitter, &row, cols, &emit_row,
prof_stats_on, mutex, col_mutex64, col_mutex32, is_gap);
}
emitter_json_array_end(emitter); /* Close "bins". */
emitter_table_sparse_end(emitter);
}
typedef struct {
const stats_arena_lextent_t *lextent;
unsigned ind;
uint64_t uptime;
} stats_arena_lextent_emit_row_t;
#define LEXTENT_COL_GET(name, value_member, field) \
static void \
stats_lextent_col_get_##name(const void *vrow, emitter_col_t *col) { \
const stats_arena_lextent_emit_row_t *row = vrow; \
col->value_member = row->lextent->field; \
}
#define LEXTENT_COL_GET_RATE(name, field) \
static void \
stats_lextent_col_get_##name(const void *vrow, emitter_col_t *col) { \
const stats_arena_lextent_emit_row_t *row = vrow; \
col->uint64_val = rate_per_second( \
row->lextent->field, row->uptime); \
}
LEXTENT_COL_GET(size, size_val, lextent_size)
LEXTENT_COL_GET(nmalloc, uint64_val, nmalloc)
LEXTENT_COL_GET(ndalloc, uint64_val, ndalloc)
LEXTENT_COL_GET(nrequests, uint64_val, nrequests)
LEXTENT_COL_GET(prof_live_requested, uint64_val, prof_live.req_sum)
LEXTENT_COL_GET(prof_live_count, uint64_val, prof_live.count)
LEXTENT_COL_GET(prof_accum_requested, uint64_val, prof_accum.req_sum)
LEXTENT_COL_GET(prof_accum_count, uint64_val, prof_accum.count)
LEXTENT_COL_GET(curlextents, size_val, curlextents)
LEXTENT_COL_GET_RATE(nmalloc_ps, nmalloc)
LEXTENT_COL_GET_RATE(ndalloc_ps, ndalloc)
LEXTENT_COL_GET_RATE(nrequests_ps, nrequests)
#undef LEXTENT_COL_GET
#undef LEXTENT_COL_GET_RATE
static void
stats_lextent_col_get_ind(const void *vrow, emitter_col_t *col) {
const stats_arena_lextent_emit_row_t *row = vrow;
col->unsigned_val = row->ind;
}
static void
stats_lextent_col_get_allocated(const void *vrow, emitter_col_t *col) {
const stats_arena_lextent_emit_row_t *row = vrow;
col->size_val = row->lextent->curlextents * row->lextent->lextent_size;
}
#define LEXTENT_COL_SIZE 0
#define LEXTENT_DESC(key, label, width, type, flags, name) \
{key, label, emitter_justify_right, width, emitter_type_##type, flags, \
stats_lextent_col_get_##name}
static const emitter_col_desc_t stats_lextent_cols[] = {
LEXTENT_DESC("size", "size", 20, size, STATS_COL_FLAG_NONE, size),
LEXTENT_DESC("ind", "ind", 4, unsigned, STATS_COL_FLAG_NONE, ind),
LEXTENT_DESC("allocated", "allocated", 13, size, STATS_COL_FLAG_NONE,
allocated),
LEXTENT_DESC("nmalloc", "nmalloc", 13, uint64, STATS_COL_FLAG_NONE,
nmalloc),
LEXTENT_DESC("nmalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE,
nmalloc_ps),
LEXTENT_DESC("ndalloc", "ndalloc", 13, uint64, STATS_COL_FLAG_NONE,
ndalloc),
LEXTENT_DESC("ndalloc_ps", "(#/sec)", 8, uint64, STATS_COL_FLAG_NONE,
ndalloc_ps),
LEXTENT_DESC("nrequests", "nrequests", 13, uint64,
STATS_COL_FLAG_NONE,
nrequests),
LEXTENT_DESC("nrequests_ps", "(#/sec)", 8, uint64,
STATS_COL_FLAG_NONE,
nrequests_ps),
LEXTENT_DESC("prof_live_requested", "prof_live_requested", 21, uint64,
STATS_COL_FLAG_PROF, prof_live_requested),
LEXTENT_DESC("prof_live_count", "prof_live_count", 17, uint64,
STATS_COL_FLAG_PROF, prof_live_count),
LEXTENT_DESC("prof_accum_requested", "prof_accum_requested", 21,
uint64, STATS_COL_FLAG_PROF, prof_accum_requested),
LEXTENT_DESC("prof_accum_count", "prof_accum_count", 17, uint64,
STATS_COL_FLAG_PROF, prof_accum_count),
LEXTENT_DESC("curlextents", "curlextents", 13, size,
STATS_COL_FLAG_NONE, curlextents),
};
#undef LEXTENT_DESC
#define LEXTENT_COL_COUNT \
(sizeof(stats_lextent_cols) / sizeof(stats_lextent_cols[0]))
static void
stats_emit_arena_lextent_row(emitter_t *emitter, emitter_row_t *table_row,
emitter_col_t *cols, const stats_arena_lextent_emit_row_t *row,
bool prof_stats_on, size_t prev_size, bool is_gap) {
unsigned active_flags = stats_col_active_flags(prof_stats_on);
emitter_col_table_fill(stats_lextent_cols, LEXTENT_COL_COUNT,
active_flags, cols, row);
emitter_json_object_begin(emitter);
emitter_col_table_emit_json(emitter, stats_lextent_cols,
LEXTENT_COL_COUNT, active_flags, cols);
emitter_json_object_end(emitter);
char size_buf[48];
stats_size_col_set(&cols[LEXTENT_COL_SIZE], row->lextent->lextent_size,
prev_size, size_buf, sizeof(size_buf));
emitter_table_sparse_row(emitter, table_row, is_gap);
}
JEMALLOC_COLD
static void
stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
/* Streaming table; see stats_arena_bins_print for the rationale. */
unsigned nbins, nlextents, j;
CTL_GET("arenas.nbins", &nbins, unsigned);
CTL_GET("arenas.nlextents", &nlextents, unsigned);
bool prof_stats_on = config_prof && opt_prof && opt_prof_stats
&& i == MALLCTL_ARENAS_ALL;
emitter_row_t row, header_row;
emitter_col_t cols[LEXTENT_COL_COUNT], header_cols[LEXTENT_COL_COUNT];
emitter_col_table_build(stats_lextent_cols, LEXTENT_COL_COUNT,
stats_col_active_flags(prof_stats_on), &row, cols, &header_row,
header_cols);
emitter_col_table_header(emitter, &header_row,
&header_cols[LEXTENT_COL_SIZE], "large:", "lextents");
size_t stats_arenas_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
stats_arenas_mib[2] = i;
CTL_LEAF_PREPARE(stats_arenas_mib, 3, "lextents");
size_t arenas_lextent_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(arenas_lextent_mib, 0, "arenas.lextent");
size_t prof_stats_mib[CTL_MAX_DEPTH];
if (prof_stats_on) {
CTL_LEAF_PREPARE(prof_stats_mib, 0, "prof.stats.lextents");
}
/*
* Carry the previous size class's size forward for the "(prev,size]"
* range label instead of looking it up by a large size-class index:
* with large size classes disabled (the default) that index may be out
* of range, and indexing sz_index2size_tab there would be unsafe. The
* class before the first large one is the last bin; sz_index2size_compute
* derives it arithmetically (no table access).
*/
size_t prev_size = sz_index2size_compute(nbins - 1);
emitter_table_sparse_begin(emitter);
for (j = 0; j < nlextents; j++) {
stats_arena_lextent_t lext;
stats_gather_arena_lextent(stats_arenas_mib, arenas_lextent_mib,
prof_stats_mib, j, prof_stats_on, &lext);
bool is_gap = (lext.nrequests == 0);
stats_arena_lextent_emit_row_t emit_row = {
&lext, nbins + j, uptime};
stats_emit_arena_lextent_row(emitter, &row, cols, &emit_row,
prof_stats_on, prev_size, is_gap);
prev_size = lext.lextent_size;
}
emitter_json_array_end(emitter); /* Close "lextents". */
emitter_table_sparse_end(emitter);
}
typedef struct {
const stats_arena_extent_t *extent;
unsigned ind;
size_t size;
} stats_arena_extent_emit_row_t;
#define EXTENT_COL_GET(name, field) \
static void \
stats_extent_col_get_##name(const void *vrow, emitter_col_t *col) { \
const stats_arena_extent_emit_row_t *row = vrow; \
col->size_val = row->extent->field; \
}
EXTENT_COL_GET(ndirty, ndirty)
EXTENT_COL_GET(dirty, dirty_bytes)
EXTENT_COL_GET(nmuzzy, nmuzzy)
EXTENT_COL_GET(muzzy, muzzy_bytes)
EXTENT_COL_GET(nretained, nretained)
EXTENT_COL_GET(retained, retained_bytes)
EXTENT_COL_GET(npinned, npinned)
EXTENT_COL_GET(pinned, pinned_bytes)
EXTENT_COL_GET(ntotal, ntotal)
EXTENT_COL_GET(total, total_bytes)
#undef EXTENT_COL_GET
static void
stats_extent_col_get_size(const void *vrow, emitter_col_t *col) {
const stats_arena_extent_emit_row_t *row = vrow;
col->size_val = row->size;
}
static void
stats_extent_col_get_ind(const void *vrow, emitter_col_t *col) {
const stats_arena_extent_emit_row_t *row = vrow;
col->unsigned_val = row->ind;
}
#define EXTENT_COL_SIZE 0
#define EXTENT_DESC(key, label, name) \
{key, label, emitter_justify_right, 13, emitter_type_size, \
STATS_COL_FLAG_NONE, stats_extent_col_get_##name}
static const emitter_col_desc_t stats_extent_cols[] = {
{"size", "size", emitter_justify_right, 20, emitter_type_size,
STATS_COL_FLAG_NONE, stats_extent_col_get_size},
{"ind", "ind", emitter_justify_right, 4, emitter_type_unsigned,
STATS_COL_FLAG_NONE, stats_extent_col_get_ind},
EXTENT_DESC("ndirty", "ndirty", ndirty),
EXTENT_DESC("dirty_bytes", "dirty", dirty),
EXTENT_DESC("nmuzzy", "nmuzzy", nmuzzy),
EXTENT_DESC("muzzy_bytes", "muzzy", muzzy),
EXTENT_DESC("nretained", "nretained", nretained),
EXTENT_DESC("retained_bytes", "retained", retained),
EXTENT_DESC("npinned", "npinned", npinned),
EXTENT_DESC("pinned_bytes", "pinned", pinned),
EXTENT_DESC("ntotal", "ntotal", ntotal),
EXTENT_DESC("total_bytes", "total", total),
};
#undef EXTENT_DESC
#define EXTENT_COL_COUNT \
(sizeof(stats_extent_cols) / sizeof(stats_extent_cols[0]))
static void
stats_emit_arena_extent_row(emitter_t *emitter, emitter_row_t *table_row,
emitter_col_t *cols, const stats_arena_extent_emit_row_t *row,
size_t prev_size, bool is_gap) {
emitter_col_table_fill(stats_extent_cols, EXTENT_COL_COUNT,
STATS_COL_FLAG_NONE, cols, row);
emitter_json_object_begin(emitter);
emitter_col_table_emit_json(emitter, stats_extent_cols, EXTENT_COL_COUNT,
STATS_COL_FLAG_NONE, cols);
emitter_json_object_end(emitter);
char size_buf[48];
stats_size_col_set(&cols[EXTENT_COL_SIZE], row->size, prev_size,
size_buf, sizeof(size_buf));
emitter_table_sparse_row(emitter, table_row, is_gap);
}
JEMALLOC_COLD
static void
stats_arena_extents_print(emitter_t *emitter, unsigned i) {
/* Streaming table; see stats_arena_bins_print for the rationale. */
unsigned j;
emitter_row_t row, header_row;
emitter_col_t cols[EXTENT_COL_COUNT], header_cols[EXTENT_COL_COUNT];
emitter_col_table_build(stats_extent_cols, EXTENT_COL_COUNT,
STATS_COL_FLAG_NONE, &row, cols, &header_row, header_cols);
emitter_col_table_header(emitter, &header_row,
&header_cols[EXTENT_COL_SIZE], "extents:", "extents");
size_t stats_arenas_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
stats_arenas_mib[2] = i;
CTL_LEAF_PREPARE(stats_arenas_mib, 3, "extents");
emitter_table_sparse_begin(emitter);
for (j = 0; j < SC_NPSIZES; j++) {
stats_arena_extent_t e;
stats_gather_arena_extent(stats_arenas_mib, j, &e);
bool is_gap = (e.ntotal == 0);
stats_arena_extent_emit_row_t emit_row = {&e, j, sz_pind2sz(j)};
stats_emit_arena_extent_row(emitter, &row, cols, &emit_row,
j > 0 ? sz_pind2sz(j - 1) : 0, is_gap);
}
emitter_json_array_end(emitter); /* Close "extents". */
emitter_table_sparse_end(emitter);
}
static void
stats_emit_arena_hpa_sec(emitter_t *emitter, const stats_arena_hpa_sec_t *sec) {
emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache",
emitter_type_size, &sec->sec_bytes);
emitter_kv(emitter, "sec_hits", "Total hits in small extent cache",
emitter_type_size, &sec->sec_hits);
emitter_kv(emitter, "sec_misses", "Total misses in small extent cache",
emitter_type_size, &sec->sec_misses);
emitter_kv(emitter, "sec_dalloc_noflush",
"Dalloc calls without flush in small extent cache",
emitter_type_size, &sec->sec_dalloc_noflush);
emitter_kv(emitter, "sec_dalloc_flush",
"Dalloc calls with flush in small extent cache", emitter_type_size,
&sec->sec_dalloc_flush);
emitter_kv(emitter, "sec_overfills",
"sec_fill calls that went over max_bytes", emitter_type_size,
&sec->sec_overfills);
}
static void
stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) {
stats_arena_hpa_sec_t sec;
stats_gather_arena_hpa_sec(i, &sec);
stats_emit_arena_hpa_sec(emitter, &sec);
}
static void
stats_emit_arena_pac_sec(emitter_t *emitter, const stats_arena_pac_sec_t *sec) {
emitter_kv(emitter, "pac_sec_bytes", "Bytes in PAC small extent cache",
emitter_type_size, &sec->sec_bytes);
emitter_kv(emitter, "pac_sec_hits", "Total hits in PAC small extent cache",
emitter_type_size, &sec->sec_hits);
emitter_kv(emitter, "pac_sec_misses",
"Total misses in PAC small extent cache", emitter_type_size,
&sec->sec_misses);
emitter_kv(emitter, "pac_sec_dalloc_noflush",
"Dalloc calls without flush in PAC small extent cache",
emitter_type_size, &sec->sec_dalloc_noflush);
emitter_kv(emitter, "pac_sec_dalloc_flush",
"Dalloc calls with flush in PAC small extent cache",
emitter_type_size, &sec->sec_dalloc_flush);
}
static void
stats_arena_pac_sec_print(emitter_t *emitter, unsigned i) {
stats_arena_pac_sec_t sec;
stats_gather_arena_pac_sec(i, &sec);
stats_emit_arena_pac_sec(emitter, &sec);
}
static void
stats_emit_arena_hpa_counters(emitter_t *emitter,
const stats_arena_hpa_counters_t *c, uint64_t uptime) {
/* Merged pageslab / page counts (broken down by huginess below). */
emitter_kv(emitter, "npageslabs", "npageslabs", emitter_type_size,
&c->npageslabs);
emitter_kv(emitter, "nactive", "nactive", emitter_type_size,
&c->nactive);
emitter_kv(emitter, "ndirty", "ndirty", emitter_type_size, &c->ndirty);
uint64_t npurge_passes_ps = rate_per_second(c->npurge_passes, uptime);
uint64_t npurges_ps = rate_per_second(c->npurges, uptime);
uint64_t nhugifies_ps = rate_per_second(c->nhugifies, uptime);
uint64_t nhugify_failures_ps =
rate_per_second(c->nhugify_failures, uptime);
uint64_t ndehugifies_ps = rate_per_second(c->ndehugifies, uptime);
emitter_kv_note(emitter, "npurge_passes", "npurge_passes",
emitter_type_uint64, &c->npurge_passes, "per_sec",
emitter_type_uint64, &npurge_passes_ps);
emitter_kv_note(emitter, "npurges", "npurges", emitter_type_uint64,
&c->npurges, "per_sec", emitter_type_uint64, &npurges_ps);
emitter_kv_note(emitter, "nhugifies", "nhugifies", emitter_type_uint64,
&c->nhugifies, "per_sec", emitter_type_uint64, &nhugifies_ps);
emitter_kv_note(emitter, "nhugify_failures", "nhugify_failures",
emitter_type_uint64, &c->nhugify_failures, "per_sec",
emitter_type_uint64, &nhugify_failures_ps);
emitter_kv_note(emitter, "ndehugifies", "ndehugifies",
emitter_type_uint64, &c->ndehugifies, "per_sec",
emitter_type_uint64, &ndehugifies_ps);
emitter_dict_begin(emitter, "slabs", "slabs");
emitter_kv(emitter, "npageslabs_nonhuge", "npageslabs_nonhuge",
emitter_type_size, &c->npageslabs_nonhuge);
emitter_kv(emitter, "nactive_nonhuge", "nactive_nonhuge",
emitter_type_size, &c->nactive_nonhuge);
emitter_kv(emitter, "ndirty_nonhuge", "ndirty_nonhuge",
emitter_type_size, &c->ndirty_nonhuge);
emitter_kv(emitter, "nretained_nonhuge", "nretained_nonhuge",
emitter_type_size, &c->nretained_nonhuge);
emitter_kv(emitter, "npageslabs_huge", "npageslabs_huge",
emitter_type_size, &c->npageslabs_huge);
emitter_kv(emitter, "nactive_huge", "nactive_huge", emitter_type_size,
&c->nactive_huge);
emitter_kv(emitter, "ndirty_huge", "ndirty_huge", emitter_type_size,
&c->ndirty_huge);
emitter_dict_end(emitter);
}
static void
stats_arena_hpa_shard_counters_print(
emitter_t *emitter, unsigned i, uint64_t uptime) {
stats_arena_hpa_counters_t c;
stats_gather_arena_hpa_counters(i, &c);
stats_emit_arena_hpa_counters(emitter, &c, uptime);
/*
* The extent-allocation distribution below is a small fixed-size
* (SEC_MAX_NALLOCS + 1) table with its own leaf ctls, so it is
* gathered into stack arrays and emitted (table then JSON) inline
* rather than through a stats_gather/stats_emit pair.
*/
uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
size_t alloc_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(alloc_mib, 0, "stats.arenas");
alloc_mib[2] = i;
CTL_LEAF_PREPARE(alloc_mib, 3, "hpa_shard.alloc");
for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
alloc_mib[5] = j;
CTL_LEAF(alloc_mib, 6, "min_extents", &hpa_alloc_min_extents[j],
uint64_t);
CTL_LEAF(alloc_mib, 6, "max_extents", &hpa_alloc_max_extents[j],
uint64_t);
CTL_LEAF(
alloc_mib, 6, "extents", &hpa_alloc_extents[j], uint64_t);
CTL_LEAF(alloc_mib, 6, "ps", &hpa_alloc_ps[j], uint64_t);
CTL_LEAF(alloc_mib, 6, "pages_per_ps",
&hpa_alloc_pages_per_ps[j], uint64_t);
CTL_LEAF(alloc_mib, 6, "extents_per_ps",
&hpa_alloc_extents_per_ps[j], uint64_t);
CTL_LEAF(alloc_mib, 6, "total_elapsed_ns_per_ps",
&hpa_alloc_total_elapsed_ns_per_ps[j], uint64_t);
}
emitter_table_printf(emitter, " extent allocation distribution:\n");
emitter_table_printf(emitter,
" %4s %20s %20s %20s %20s %20s %20s %24s %24s\n", "",
"min_extents", "max_extents",
"extents", "ps", "pages_per_ps", "extents_per_ps",
"total_elapsed_ns_per_ps", "elapsed_ns_per_ps");
for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
const uint64_t extents_per_ps = hpa_alloc_extents_per_ps[j];
const uint64_t total_elapsed_ns_per_ps =
hpa_alloc_total_elapsed_ns_per_ps[j];
const uint64_t elapsed_ns_per_ps = (extents_per_ps != 0)
? (total_elapsed_ns_per_ps / extents_per_ps)
: 0;
emitter_table_printf(emitter,
" %4zu %20" FMTu64 " %20" FMTu64 " %20" FMTu64
" %20" FMTu64 " %20" FMTu64 " %20" FMTu64 " %24" FMTu64
" %24" FMTu64 "\n",
j, hpa_alloc_min_extents[j], hpa_alloc_max_extents[j],
hpa_alloc_extents[j],
hpa_alloc_ps[j], hpa_alloc_pages_per_ps[j], extents_per_ps,
total_elapsed_ns_per_ps, elapsed_ns_per_ps);
}
emitter_table_printf(emitter, "\n");
emitter_json_array_kv_begin(emitter, "extent_allocation_distribution");
for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "min_extents", emitter_type_uint64,
&hpa_alloc_min_extents[j]);
emitter_json_kv(emitter, "max_extents", emitter_type_uint64,
&hpa_alloc_max_extents[j]);
emitter_json_kv(emitter, "extents", emitter_type_uint64,
&hpa_alloc_extents[j]);
emitter_json_kv(
emitter, "ps", emitter_type_uint64, &hpa_alloc_ps[j]);
emitter_json_kv(emitter, "pages_per_ps", emitter_type_uint64,
&hpa_alloc_pages_per_ps[j]);
emitter_json_kv(emitter, "extents_per_ps", emitter_type_uint64,
&hpa_alloc_extents_per_ps[j]);
emitter_json_kv(emitter, "total_elapsed_ns_per_ps",
emitter_type_uint64, &hpa_alloc_total_elapsed_ns_per_ps[j]);
emitter_json_object_end(emitter);
}
emitter_json_array_end(emitter); /* End "alloc_batch" */
}
typedef struct {
const stats_arena_hpa_slab_t *slab;
const char *size_title;
size_t size;
size_t prev_size;
unsigned ind;
} stats_arena_hpa_slab_emit_row_t;
#define HPA_SLAB_COL_GET(name) \
static void \
stats_hpa_slab_col_get_##name(const void *vrow, emitter_col_t *col) { \
const stats_arena_hpa_slab_emit_row_t *row = vrow; \
col->size_val = row->slab->name; \
}
HPA_SLAB_COL_GET(npageslabs_huge)
HPA_SLAB_COL_GET(nactive_huge)
HPA_SLAB_COL_GET(ndirty_huge)
HPA_SLAB_COL_GET(npageslabs_nonhuge)
HPA_SLAB_COL_GET(nactive_nonhuge)
HPA_SLAB_COL_GET(ndirty_nonhuge)
HPA_SLAB_COL_GET(nretained_nonhuge)
#undef HPA_SLAB_COL_GET
static void
stats_hpa_slab_col_get_size(const void *vrow, emitter_col_t *col) {
const stats_arena_hpa_slab_emit_row_t *row = vrow;
if (row->size_title != NULL) {
col->type = emitter_type_title;
col->str_val = row->size_title;
} else {
col->size_val = row->size;
}
}
static void
stats_hpa_slab_col_get_ind(const void *vrow, emitter_col_t *col) {
const stats_arena_hpa_slab_emit_row_t *row = vrow;
if (row->size_title != NULL) {
col->type = emitter_type_title;
col->str_val = "-";
} else {
col->unsigned_val = row->ind;
}
}
#define HPA_SLAB_COL_SIZE 0
#define HPA_SLAB_DESC(name, width) \
{#name, #name, emitter_justify_right, width, emitter_type_size, \
STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_##name}
static const emitter_col_desc_t stats_hpa_slab_cols[] = {
{"size", "size", emitter_justify_right, 20, emitter_type_size,
STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_size},
{"ind", "ind", emitter_justify_right, 4, emitter_type_unsigned,
STATS_COL_FLAG_NONE, stats_hpa_slab_col_get_ind},
HPA_SLAB_DESC(npageslabs_huge, 16),
HPA_SLAB_DESC(nactive_huge, 16),
HPA_SLAB_DESC(ndirty_huge, 16),
HPA_SLAB_DESC(npageslabs_nonhuge, 20),
HPA_SLAB_DESC(nactive_nonhuge, 20),
HPA_SLAB_DESC(ndirty_nonhuge, 20),
HPA_SLAB_DESC(nretained_nonhuge, 20),
};
#undef HPA_SLAB_DESC
#define HPA_SLAB_COL_COUNT \
(sizeof(stats_hpa_slab_cols) / sizeof(stats_hpa_slab_cols[0]))
static void
stats_emit_arena_hpa_slab_row(emitter_t *emitter,
emitter_row_t *table_row, emitter_col_t *cols,
const stats_arena_hpa_slab_emit_row_t *row, const char *json_key,
bool sparse, bool is_gap) {
emitter_col_table_fill(stats_hpa_slab_cols, HPA_SLAB_COL_COUNT,
STATS_COL_FLAG_NONE, cols, row);
if (json_key != NULL) {
emitter_json_object_kv_begin(emitter, json_key);
} else {
emitter_json_object_begin(emitter);
}
emitter_col_table_emit_json(emitter, stats_hpa_slab_cols,
HPA_SLAB_COL_COUNT, STATS_COL_FLAG_NONE, cols);
emitter_json_object_end(emitter);
char size_buf[48];
if (row->size_title == NULL) {
stats_size_col_set(&cols[HPA_SLAB_COL_SIZE], row->size,
row->prev_size, size_buf, sizeof(size_buf));
}
if (sparse) {
emitter_table_sparse_row(emitter, table_row, is_gap);
} else {
emitter_table_row(emitter, table_row);
}
}
static void
stats_arena_hpa_shard_slabs_print(emitter_t *emitter, unsigned i) {
emitter_row_t row, header_row;
emitter_col_t cols[HPA_SLAB_COL_COUNT];
emitter_col_t header_cols[HPA_SLAB_COL_COUNT];
emitter_col_table_build(stats_hpa_slab_cols, HPA_SLAB_COL_COUNT,
STATS_COL_FLAG_NONE, &row, cols, &header_row, header_cols);
emitter_table_printf(emitter, "pageslabs:\n");
emitter_table_row(emitter, &header_row);
/*
* Full and empty pageslabs are shown as two symbolic rows (size =
* "full"/"empty", no size-class index) at the top of the table; their
* JSON stays in the separate full_slabs / empty_slabs objects.
*/
stats_arena_hpa_slab_t sfull;
stats_gather_arena_hpa_slab(i, "full_slabs", &sfull);
stats_arena_hpa_slab_emit_row_t full_row = {
&sfull, "full", 0, 0, 0};
stats_emit_arena_hpa_slab_row(emitter, &row, cols, &full_row,
"full_slabs", false, false);
stats_arena_hpa_slab_t sempty;
stats_gather_arena_hpa_slab(i, "empty_slabs", &sempty);
stats_arena_hpa_slab_emit_row_t empty_row = {
&sempty, "empty", 0, 0, 0};
stats_emit_arena_hpa_slab_row(emitter, &row, cols, &empty_row,
"empty_slabs", false, false);
size_t stats_arenas_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
stats_arenas_mib[2] = i;
CTL_LEAF_PREPARE(stats_arenas_mib, 3, "hpa_shard.nonfull_slabs");
emitter_json_array_kv_begin(emitter, "nonfull_slabs");
emitter_table_sparse_begin(emitter);
for (pszind_t j = 0; j < PSSET_NPSIZES && j < SC_NPSIZES; j++) {
stats_arena_hpa_slab_t s;
stats_gather_arena_hpa_nonfull(stats_arenas_mib, j, &s);
bool is_gap = (s.npageslabs_huge == 0 && s.npageslabs_nonhuge == 0);
stats_arena_hpa_slab_emit_row_t emit_row = {
&s, NULL, sz_pind2sz(j),
j > 0 ? sz_pind2sz(j - 1) : 0, j};
stats_emit_arena_hpa_slab_row(emitter, &row, cols, &emit_row,
NULL, true, is_gap);
}
emitter_json_array_end(emitter); /* End "nonfull_slabs" */
emitter_table_sparse_end(emitter);
}
static void
stats_arena_hpa_shard_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
emitter_json_object_kv_begin(emitter, "hpa_shard");
stats_arena_hpa_shard_sec_print(emitter, i);
stats_arena_hpa_shard_counters_print(emitter, i, uptime);
stats_arena_hpa_shard_slabs_print(emitter, i);
emitter_json_object_end(emitter); /* End "hpa_shard" */
}
static void
stats_arena_mutexes_print(
emitter_t *emitter, unsigned arena_ind, uint64_t uptime) {
/*
* Mutex readers gather counters directly into emitter columns, coupling
* collection to the output representation. Emission remains a separate
* mutex_stats_emit() call.
*/
emitter_row_t row;
emitter_col_t col_name;
emitter_col_t col64[mutex_prof_num_uint64_t_counters];
emitter_col_t col32[mutex_prof_num_uint32_t_counters];
emitter_row_init(&row);
mutex_stats_init_cols(&row, "", &col_name, col64, col32);
emitter_json_object_kv_begin(emitter, "mutexes");
emitter_table_row(emitter, &row);
size_t stats_arenas_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(stats_arenas_mib, 0, "stats.arenas");
stats_arenas_mib[2] = arena_ind;
CTL_LEAF_PREPARE(stats_arenas_mib, 3, "mutexes");
for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes;
i++) {
const char *name = arena_mutex_names[i];
emitter_json_object_kv_begin(emitter, name);
mutex_stats_read(
stats_arenas_mib, 4, name, &col_name, col64, col32, uptime);
mutex_stats_emit(emitter, &row, col64, col32);
emitter_json_object_end(emitter); /* Close the mutex dict. */
}
emitter_json_object_end(emitter); /* End "mutexes". */
}
static void
stats_emit_arena_basics(emitter_t *emitter, const stats_arena_basics_t *b) {
if (b->has_name) {
const char *namep = b->name;
emitter_kv(
emitter, "name", "name", emitter_type_string, &namep);
}
emitter_kv(emitter, "nthreads", "assigned threads",
emitter_type_unsigned, &b->nthreads);
emitter_kv(emitter, "uptime_ns", "uptime", emitter_type_uint64,
&b->uptime);
emitter_kv(emitter, "dss", "dss allocation precedence",
emitter_type_string, &b->dss);
}
static uint64_t
stats_arena_basics_print(emitter_t *emitter, unsigned i) {
stats_arena_basics_t basics;
stats_gather_arena_basics(i, &basics);
stats_emit_arena_basics(emitter, &basics);
return basics.uptime;
}
static void
stats_emit_arena_decay(emitter_t *emitter, const stats_arena_decay_t *d) {
emitter_row_t decay_row;
emitter_row_init(&decay_row);
/* JSON-style emission. */
emitter_json_kv(
emitter, "dirty_decay_ms", emitter_type_ssize, &d->dirty_decay_ms);
emitter_json_kv(
emitter, "muzzy_decay_ms", emitter_type_ssize, &d->muzzy_decay_ms);
emitter_json_kv(emitter, "pactive", emitter_type_size, &d->pactive);
emitter_json_kv(emitter, "pdirty", emitter_type_size, &d->pdirty);
emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &d->pmuzzy);
emitter_json_kv(
emitter, "dirty_npurge", emitter_type_uint64, &d->dirty_npurge);
emitter_json_kv(
emitter, "dirty_nmadvise", emitter_type_uint64, &d->dirty_nmadvise);
emitter_json_kv(
emitter, "dirty_purged", emitter_type_uint64, &d->dirty_purged);
emitter_json_kv(
emitter, "muzzy_npurge", emitter_type_uint64, &d->muzzy_npurge);
emitter_json_kv(
emitter, "muzzy_nmadvise", emitter_type_uint64, &d->muzzy_nmadvise);
emitter_json_kv(
emitter, "muzzy_purged", emitter_type_uint64, &d->muzzy_purged);
/* Table-style emission. */
COL(decay_row, decay_type, right, 9, title);
col_decay_type.str_val = "decaying:";
COL(decay_row, decay_time, right, 6, title);
col_decay_time.str_val = "time";
COL(decay_row, decay_npages, right, 13, title);
col_decay_npages.str_val = "npages";
COL(decay_row, decay_sweeps, right, 13, title);
col_decay_sweeps.str_val = "sweeps";
COL(decay_row, decay_madvises, right, 13, title);
col_decay_madvises.str_val = "madvises";
COL(decay_row, decay_purged, right, 13, title);
col_decay_purged.str_val = "purged";
/* Title row. */
emitter_table_row(emitter, &decay_row);
/* Dirty row. */
col_decay_type.str_val = "dirty:";
if (d->dirty_decay_ms >= 0) {
col_decay_time.type = emitter_type_ssize;
col_decay_time.ssize_val = d->dirty_decay_ms;
} else {
col_decay_time.type = emitter_type_title;
col_decay_time.str_val = "N/A";
}
col_decay_npages.type = emitter_type_size;
col_decay_npages.size_val = d->pdirty;
col_decay_sweeps.type = emitter_type_uint64;
col_decay_sweeps.uint64_val = d->dirty_npurge;
col_decay_madvises.type = emitter_type_uint64;
col_decay_madvises.uint64_val = d->dirty_nmadvise;
col_decay_purged.type = emitter_type_uint64;
col_decay_purged.uint64_val = d->dirty_purged;
emitter_table_row(emitter, &decay_row);
/* Muzzy row. */
col_decay_type.str_val = "muzzy:";
if (d->muzzy_decay_ms >= 0) {
col_decay_time.type = emitter_type_ssize;
col_decay_time.ssize_val = d->muzzy_decay_ms;
} else {
col_decay_time.type = emitter_type_title;
col_decay_time.str_val = "N/A";
}
col_decay_npages.type = emitter_type_size;
col_decay_npages.size_val = d->pmuzzy;
col_decay_sweeps.type = emitter_type_uint64;
col_decay_sweeps.uint64_val = d->muzzy_npurge;
col_decay_madvises.type = emitter_type_uint64;
col_decay_madvises.uint64_val = d->muzzy_nmadvise;
col_decay_purged.type = emitter_type_uint64;
col_decay_purged.uint64_val = d->muzzy_purged;
emitter_table_row(emitter, &decay_row);
}
static size_t
stats_arena_decay_print(emitter_t *emitter, unsigned i) {
stats_arena_decay_t decay;
stats_gather_arena_decay(i, &decay);
stats_emit_arena_decay(emitter, &decay);
return decay.pactive;
}
static void
stats_emit_arena_alloc(emitter_t *emitter, const stats_arena_alloc_t *small,
const stats_arena_alloc_t *large, uint64_t uptime) {
emitter_row_t alloc_count_row;
emitter_row_init(&alloc_count_row);
COL(alloc_count_row, count_title, left, 21, title);
col_count_title.str_val = "";
COL(alloc_count_row, count_allocated, right, 16, title);
col_count_allocated.str_val = "allocated";
COL(alloc_count_row, count_nmalloc, right, 16, title);
col_count_nmalloc.str_val = "nmalloc";
COL(alloc_count_row, count_nmalloc_ps, right, 10, title);
col_count_nmalloc_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_ndalloc, right, 16, title);
col_count_ndalloc.str_val = "ndalloc";
COL(alloc_count_row, count_ndalloc_ps, right, 10, title);
col_count_ndalloc_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_nrequests, right, 16, title);
col_count_nrequests.str_val = "nrequests";
COL(alloc_count_row, count_nrequests_ps, right, 10, title);
col_count_nrequests_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_nfills, right, 16, title);
col_count_nfills.str_val = "nfill";
COL(alloc_count_row, count_nfills_ps, right, 10, title);
col_count_nfills_ps.str_val = "(#/sec)";
COL(alloc_count_row, count_nflushes, right, 16, title);
col_count_nflushes.str_val = "nflush";
COL(alloc_count_row, count_nflushes_ps, right, 10, title);
col_count_nflushes_ps.str_val = "(#/sec)";
emitter_table_row(emitter, &alloc_count_row);
col_count_nmalloc_ps.type = emitter_type_uint64;
col_count_ndalloc_ps.type = emitter_type_uint64;
col_count_nrequests_ps.type = emitter_type_uint64;
col_count_nfills_ps.type = emitter_type_uint64;
col_count_nflushes_ps.type = emitter_type_uint64;
/*
* JSON keys for the value columns; emitter_row() emits these as kvs.
* The (#/sec) rate columns and the title column stay table-only (no
* json_key).
*/
col_count_allocated.json_key = "allocated";
col_count_nmalloc.json_key = "nmalloc";
col_count_ndalloc.json_key = "ndalloc";
col_count_nrequests.json_key = "nrequests";
col_count_nfills.json_key = "nfills";
col_count_nflushes.json_key = "nflushes";
#define EMIT_ALLOC_STAT(a, name, valtype) \
col_count_##name.type = emitter_type_##valtype; \
col_count_##name.valtype##_val = (a)->name;
emitter_json_object_kv_begin(emitter, "small");
col_count_title.str_val = "small:";
EMIT_ALLOC_STAT(small, allocated, size)
EMIT_ALLOC_STAT(small, nmalloc, uint64)
col_count_nmalloc_ps.uint64_val = rate_per_second(
col_count_nmalloc.uint64_val, uptime);
EMIT_ALLOC_STAT(small, ndalloc, uint64)
col_count_ndalloc_ps.uint64_val = rate_per_second(
col_count_ndalloc.uint64_val, uptime);
EMIT_ALLOC_STAT(small, nrequests, uint64)
col_count_nrequests_ps.uint64_val = rate_per_second(
col_count_nrequests.uint64_val, uptime);
EMIT_ALLOC_STAT(small, nfills, uint64)
col_count_nfills_ps.uint64_val = rate_per_second(
col_count_nfills.uint64_val, uptime);
EMIT_ALLOC_STAT(small, nflushes, uint64)
col_count_nflushes_ps.uint64_val = rate_per_second(
col_count_nflushes.uint64_val, uptime);
emitter_row(emitter, &alloc_count_row);
emitter_json_object_end(emitter); /* Close "small". */
emitter_json_object_kv_begin(emitter, "large");
col_count_title.str_val = "large:";
EMIT_ALLOC_STAT(large, allocated, size)
EMIT_ALLOC_STAT(large, nmalloc, uint64)
col_count_nmalloc_ps.uint64_val = rate_per_second(
col_count_nmalloc.uint64_val, uptime);
EMIT_ALLOC_STAT(large, ndalloc, uint64)
col_count_ndalloc_ps.uint64_val = rate_per_second(
col_count_ndalloc.uint64_val, uptime);
EMIT_ALLOC_STAT(large, nrequests, uint64)
col_count_nrequests_ps.uint64_val = rate_per_second(
col_count_nrequests.uint64_val, uptime);
EMIT_ALLOC_STAT(large, nfills, uint64)
col_count_nfills_ps.uint64_val = rate_per_second(
col_count_nfills.uint64_val, uptime);
EMIT_ALLOC_STAT(large, nflushes, uint64)
col_count_nflushes_ps.uint64_val = rate_per_second(
col_count_nflushes.uint64_val, uptime);
emitter_row(emitter, &alloc_count_row);
emitter_json_object_end(emitter); /* Close "large". */
#undef EMIT_ALLOC_STAT
/* Aggregated small + large stats are emitter only in table mode. */
col_count_title.str_val = "total:";
col_count_allocated.size_val = small->allocated + large->allocated;
col_count_nmalloc.uint64_val = small->nmalloc + large->nmalloc;
col_count_ndalloc.uint64_val = small->ndalloc + large->ndalloc;
col_count_nrequests.uint64_val = small->nrequests + large->nrequests;
col_count_nfills.uint64_val = small->nfills + large->nfills;
col_count_nflushes.uint64_val = small->nflushes + large->nflushes;
col_count_nmalloc_ps.uint64_val = rate_per_second(
col_count_nmalloc.uint64_val, uptime);
col_count_ndalloc_ps.uint64_val = rate_per_second(
col_count_ndalloc.uint64_val, uptime);
col_count_nrequests_ps.uint64_val = rate_per_second(
col_count_nrequests.uint64_val, uptime);
col_count_nfills_ps.uint64_val = rate_per_second(
col_count_nfills.uint64_val, uptime);
col_count_nflushes_ps.uint64_val = rate_per_second(
col_count_nflushes.uint64_val, uptime);
emitter_table_row(emitter, &alloc_count_row);
}
static void
stats_arena_alloc_print(emitter_t *emitter, unsigned i, uint64_t uptime) {
stats_arena_alloc_t small_alloc, large_alloc;
stats_gather_arena_alloc(i, &small_alloc, &large_alloc);
stats_emit_arena_alloc(emitter, &small_alloc, &large_alloc, uptime);
}
static void
stats_emit_arena_mem(emitter_t *emitter, const stats_arena_mem_t *mem,
size_t active_bytes) {
emitter_row_t mem_count_row;
emitter_row_init(&mem_count_row);
emitter_col_t mem_count_title;
emitter_col_init(&mem_count_title, &mem_count_row);
mem_count_title.justify = emitter_justify_left;
mem_count_title.width = 21;
mem_count_title.type = emitter_type_title;
mem_count_title.str_val = "";
emitter_col_t mem_count_val;
emitter_col_init(&mem_count_val, &mem_count_row);
mem_count_val.justify = emitter_justify_right;
mem_count_val.width = 16;
mem_count_val.type = emitter_type_title;
mem_count_val.str_val = "";
/* Blank spacer row (both columns are empty titles; table only). */
emitter_row(emitter, &mem_count_row);
mem_count_val.type = emitter_type_size;
/*
* Active count in bytes is table only: mem_count_val carries no
* json_key, so emitter_row() skips it in JSON.
*/
mem_count_title.str_val = "active:";
mem_count_val.size_val = active_bytes;
emitter_row(emitter, &mem_count_row);
/*
* Each remaining stat is one row: the value column's json_key makes it a
* "<stat>": value pair in JSON and the aligned value in the table.
*/
#define EMIT_MEM_STAT(stat) \
mem_count_title.str_val = #stat ":"; \
mem_count_val.json_key = #stat; \
mem_count_val.size_val = mem->stat; \
emitter_row(emitter, &mem_count_row);
EMIT_MEM_STAT(mapped)
EMIT_MEM_STAT(retained)
EMIT_MEM_STAT(pinned)
EMIT_MEM_STAT(base)
EMIT_MEM_STAT(internal)
EMIT_MEM_STAT(metadata_edata)
EMIT_MEM_STAT(metadata_rtree)
EMIT_MEM_STAT(metadata_thp)
EMIT_MEM_STAT(tcache_bytes)
EMIT_MEM_STAT(tcache_stashed_bytes)
EMIT_MEM_STAT(resident)
EMIT_MEM_STAT(abandoned_vm)
EMIT_MEM_STAT(extent_avail)
#undef EMIT_MEM_STAT
}
static void
stats_arena_mem_print(emitter_t *emitter, unsigned i, size_t pactive) {
stats_arena_mem_t m;
stats_gather_arena_mem(i, &m);
stats_emit_arena_mem(emitter, &m, pactive * m.page);
}
JEMALLOC_COLD
static void
stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
bool mutex, bool extents, bool hpa) {
uint64_t uptime = stats_arena_basics_print(emitter, i);
size_t pactive = stats_arena_decay_print(emitter, i);
stats_arena_alloc_print(emitter, i, uptime);
stats_arena_mem_print(emitter, i, pactive);
if (opt_pac_sec_opts.nshards > 0) {
stats_arena_pac_sec_print(emitter, i);
}
if (mutex) {
stats_arena_mutexes_print(emitter, i, uptime);
}
if (bins) {
stats_arena_bins_print(emitter, mutex, i, uptime);
}
if (large) {
stats_arena_lextents_print(emitter, i, uptime);
}
if (extents) {
stats_arena_extents_print(emitter, i);
}
if (hpa) {
stats_arena_hpa_shard_print(emitter, i, uptime);
}
}
static void
stats_general_version(emitter_t *emitter) {
const char *cpv;
CTL_GET("version", &cpv, const char *);
emitter_kv(emitter, "version", "Version", emitter_type_string, &cpv);
}
static void
stats_general_config(emitter_t *emitter) {
/*
* Most config entries are independent scalars that
* CONFIG_WRITE_BOOL reads and emits together. Keeping those operations
* local is clearer than introducing a gather struct for this flat list.
*/
bool bv;
emitter_dict_begin(emitter, "config", "Build-time option settings");
CONFIG_WRITE_BOOL(cache_oblivious);
CONFIG_WRITE_BOOL(debug);
CONFIG_WRITE_BOOL(fill);
CONFIG_WRITE_BOOL(infallible_new);
CONFIG_WRITE_BOOL(lazy_lock);
emitter_kv(emitter, "malloc_conf", "config.malloc_conf",
emitter_type_string, &config_malloc_conf);
CONFIG_WRITE_BOOL(opt_safety_checks);
CONFIG_WRITE_BOOL(prof);
CONFIG_WRITE_BOOL(prof_libgcc);
CONFIG_WRITE_BOOL(prof_libunwind);
CONFIG_WRITE_BOOL(prof_frameptr);
CONFIG_WRITE_BOOL(stats);
CONFIG_WRITE_BOOL(utrace);
CONFIG_WRITE_BOOL(xmalloc);
emitter_dict_end(emitter); /* Close "config" dict. */
}
static void
stats_general_system(emitter_t *emitter) {
emitter_dict_begin(emitter, "system", "System configuration");
/*
* This shows system's THP mode detected at jemalloc's init time.
* jemalloc does not re-detect the mode even if it changes after
* jemalloc's init. It is assumed that system's THP mode is stable
* during the process's lifetime and a violation could lead to
* undefined behavior.
*/
const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode];
emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string,
&thp_mode_name);
emitter_dict_end(emitter); /* Close "system". */
}
static void
stats_general_opts(emitter_t *emitter) {
/*
* The OPT_WRITE_* macros conditionally read and emit most options.
* Options needing special formatting or availability checks stay
* explicit below.
*/
const char *cpv;
bool bv, bv2;
unsigned uv;
uint32_t u32v;
uint64_t u64v;
int64_t i64v;
ssize_t ssv, ssv2;
size_t sv, bsz, usz, u32sz, u64sz, i64sz, ssz, sssz, cpsz;
bsz = sizeof(bool);
usz = sizeof(unsigned);
ssz = sizeof(size_t);
sssz = sizeof(ssize_t);
cpsz = sizeof(const char *);
u32sz = sizeof(uint32_t);
i64sz = sizeof(int64_t);
u64sz = sizeof(uint64_t);
emitter_dict_begin(emitter, "opt", "Run-time option settings");
/*
* opt.malloc_conf.
*
* Sources are documented in https://jemalloc.net/jemalloc.3.html#tuning
* - (Not Included Here) The string specified via --with-malloc-conf,
* which is already printed out above as config.malloc_conf
* - (Included) The string pointed to by the global variable malloc_conf
* - (Included) The “name” of the file referenced by the symbolic link
* named /etc/malloc.conf
* - (Included) The value of the environment variable MALLOC_CONF
* - (Optional, Unofficial) The string pointed to by the global variable
* malloc_conf_2_conf_harder, which is hidden from the public.
*
* Note: The outputs are strictly ordered by priorities (low -> high).
*
*/
#define MALLOC_CONF_WRITE(name, message) \
if (je_mallctl("opt.malloc_conf." name, (void *)&cpv, &cpsz, NULL, 0) \
!= 0) { \
cpv = ""; \
} \
emitter_kv(emitter, name, message, emitter_type_string, &cpv);
MALLOC_CONF_WRITE("global_var", "Global variable malloc_conf");
MALLOC_CONF_WRITE("symlink", "Symbolic link malloc.conf");
MALLOC_CONF_WRITE("env_var", "Environment variable MALLOC_CONF");
/* As this config is unofficial, skip the output if it's NULL. */
if (je_malloc_conf_2_conf_harder != NULL) {
cpv = je_malloc_conf_2_conf_harder;
emitter_kv(emitter, "global_var_2_conf_harder",
"Global "
"variable malloc_conf_2_conf_harder",
emitter_type_string, &cpv);
}
#undef MALLOC_CONF_WRITE
OPT_WRITE_BOOL("abort")
OPT_WRITE_BOOL("abort_conf")
OPT_WRITE_BOOL("cache_oblivious")
OPT_WRITE_BOOL("confirm_conf")
OPT_WRITE_BOOL("experimental_hpa_start_huge_if_thp_always")
OPT_WRITE_BOOL("experimental_hpa_enforce_hugify")
OPT_WRITE_BOOL("retain")
OPT_WRITE_CHAR_P("dss")
OPT_WRITE_UNSIGNED("narenas")
OPT_WRITE_CHAR_P("percpu_arena")
OPT_WRITE_SIZE_T("oversize_threshold")
OPT_WRITE_BOOL("hpa")
OPT_WRITE_SIZE_T("hpa_slab_max_alloc")
OPT_WRITE_SIZE_T("hpa_hugification_threshold")
OPT_WRITE_UINT64("hpa_hugify_delay_ms")
OPT_WRITE_BOOL("hpa_hugify_sync")
OPT_WRITE_UINT64("hpa_min_purge_interval_ms")
if (je_mallctl("opt.hpa_dirty_mult", (void *)&u32v, &u32sz, NULL, 0)
== 0) {
/*
* We cheat a little and "know" the secret meaning of this
* representation.
*/
if (u32v == (uint32_t)-1) {
const char *neg1 = "-1";
emitter_kv(emitter, "hpa_dirty_mult",
"opt.hpa_dirty_mult", emitter_type_string, &neg1);
} else {
char buf[FXP_BUF_SIZE];
fxp_print(u32v, buf);
const char *bufp = buf;
emitter_kv(emitter, "hpa_dirty_mult",
"opt.hpa_dirty_mult", emitter_type_string, &bufp);
}
}
OPT_WRITE_SIZE_T("hpa_purge_threshold")
OPT_WRITE_UINT64("hpa_min_purge_delay_ms")
OPT_WRITE_CHAR_P("hpa_hugify_style")
OPT_WRITE_SIZE_T("hpa_sec_nshards")
OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
OPT_WRITE_SIZE_T("experimental_pac_sec_nshards")
OPT_WRITE_SIZE_T("experimental_pac_sec_max_alloc")
OPT_WRITE_SIZE_T("experimental_pac_sec_max_bytes")
OPT_WRITE_BOOL("huge_arena_pac_thp")
OPT_WRITE_CHAR_P("metadata_thp")
OPT_WRITE_INT64("mutex_max_spin")
OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread")
OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms")
OPT_WRITE_SSIZE_T_MUTABLE("muzzy_decay_ms", "arenas.muzzy_decay_ms")
OPT_WRITE_SIZE_T("lg_extent_max_active_fit")
OPT_WRITE_CHAR_P("junk")
OPT_WRITE_BOOL("zero")
OPT_WRITE_BOOL("utrace")
OPT_WRITE_BOOL("xmalloc")
OPT_WRITE_BOOL("experimental_tcache_gc")
OPT_WRITE_BOOL("tcache")
OPT_WRITE_SIZE_T("tcache_max")
OPT_WRITE_UNSIGNED("tcache_nslots_small_min")
OPT_WRITE_UNSIGNED("tcache_nslots_small_max")
OPT_WRITE_UNSIGNED("tcache_nslots_large")
OPT_WRITE_SSIZE_T("lg_tcache_nslots_mul")
OPT_WRITE_SIZE_T("tcache_gc_incr_bytes")
OPT_WRITE_SIZE_T("tcache_gc_delay_bytes")
OPT_WRITE_UNSIGNED("lg_tcache_flush_small_div")
OPT_WRITE_UNSIGNED("lg_tcache_flush_large_div")
OPT_WRITE_UNSIGNED("debug_double_free_max_scan")
OPT_WRITE_CHAR_P("thp")
OPT_WRITE_BOOL("prof")
OPT_WRITE_UNSIGNED("prof_bt_max")
OPT_WRITE_CHAR_P("prof_prefix")
OPT_WRITE_BOOL_MUTABLE("prof_active", "prof.active")
OPT_WRITE_BOOL_MUTABLE(
"prof_thread_active_init", "prof.thread_active_init")
OPT_WRITE_SSIZE_T_MUTABLE("lg_prof_sample", "prof.lg_sample")
OPT_WRITE_BOOL("prof_accum")
OPT_WRITE_SSIZE_T("lg_prof_interval")
OPT_WRITE_BOOL("prof_gdump")
OPT_WRITE_BOOL("prof_final")
OPT_WRITE_BOOL("prof_leak")
OPT_WRITE_BOOL("prof_leak_error")
OPT_WRITE_BOOL("stats_print")
OPT_WRITE_CHAR_P("stats_print_opts")
OPT_WRITE_INT64("stats_interval")
OPT_WRITE_CHAR_P("stats_interval_opts")
OPT_WRITE_CHAR_P("zero_realloc")
OPT_WRITE_SIZE_T("process_madvise_max_batch")
OPT_WRITE_BOOL("disable_large_size_classes")
emitter_dict_end(emitter); /* Close "opt". */
}
static void
stats_general_prof(emitter_t *emitter) {
if (config_prof) {
bool bv;
uint64_t u64v;
ssize_t ssv;
emitter_dict_begin(emitter, "prof", "Profiling settings");
CTL_GET("prof.thread_active_init", &bv, bool);
emitter_kv(emitter, "thread_active_init",
"prof.thread_active_init", emitter_type_bool, &bv);
CTL_GET("prof.active", &bv, bool);
emitter_kv(
emitter, "active", "prof.active", emitter_type_bool, &bv);
CTL_GET("prof.gdump", &bv, bool);
emitter_kv(
emitter, "gdump", "prof.gdump", emitter_type_bool, &bv);
CTL_GET("prof.interval", &u64v, uint64_t);
emitter_kv(emitter, "interval", "prof.interval",
emitter_type_uint64, &u64v);
CTL_GET("prof.lg_sample", &ssv, ssize_t);
emitter_kv(emitter, "lg_sample", "prof.lg_sample",
emitter_type_ssize, &ssv);
emitter_dict_end(emitter); /* Close "prof". */
}
}
static void
stats_general_bin_meta_print(emitter_t *emitter, unsigned nbins) {
/*
* Streaming JSON-only size-class metadata table (per-row gather+emit);
* see stats_arena_bins_print for the streaming rationale.
*/
emitter_json_array_kv_begin(emitter, "bin");
size_t mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(mib, 0, "arenas.bin");
for (unsigned i = 0; i < nbins; i++) {
stats_arena_bin_meta_t bm;
stats_gather_arena_bin_meta(mib, i, &bm);
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "size", emitter_type_size, &bm.size);
emitter_json_kv(emitter, "nregs", emitter_type_uint32, &bm.nregs);
emitter_json_kv(
emitter, "slab_size", emitter_type_size, &bm.slab_size);
emitter_json_kv(emitter, "nshards", emitter_type_uint32, &bm.nshards);
emitter_json_object_end(emitter);
}
emitter_json_array_end(emitter); /* Close "bin". */
}
static void
stats_general_lextent_meta_print(emitter_t *emitter, unsigned nlextents) {
/* Streaming JSON-only metadata table; see stats_general_bin_meta_print. */
emitter_json_array_kv_begin(emitter, "lextent");
size_t mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(mib, 0, "arenas.lextent");
for (unsigned i = 0; i < nlextents; i++) {
stats_arena_lextent_meta_t lm;
stats_gather_arena_lextent_meta(mib, i, &lm);
emitter_json_object_begin(emitter);
emitter_json_kv(emitter, "size", emitter_type_size, &lm.size);
emitter_json_object_end(emitter);
}
emitter_json_array_end(emitter); /* Close "lextent". */
}
static void
stats_general_arenas_print(emitter_t *emitter, bool omit_size_class_meta) {
stats_arena_config_t cfg;
stats_gather_arena_config(&cfg);
/*
* The json output sticks arena info into an "arenas" dict; the table
* output puts them at the top-level.
*/
emitter_json_object_kv_begin(emitter, "arenas");
emitter_kv(
emitter, "narenas", "Arenas", emitter_type_unsigned, &cfg.narenas);
/*
* Decay settings are emitted only in json mode; in table mode, they
* are emitted as notes with the opt output, above.
*/
emitter_json_kv(
emitter, "dirty_decay_ms", emitter_type_ssize, &cfg.dirty_decay_ms);
emitter_json_kv(
emitter, "muzzy_decay_ms", emitter_type_ssize, &cfg.muzzy_decay_ms);
emitter_kv(emitter, "quantum", "Quantum size", emitter_type_size,
&cfg.quantum);
emitter_kv(emitter, "page", "Page size", emitter_type_size, &cfg.page);
emitter_kv(emitter, "hugepage", "Hugepage size", emitter_type_size,
&cfg.hugepage);
if (cfg.have_tcache_max) {
emitter_kv(emitter, "tcache_max",
"Maximum thread-cached size class", emitter_type_size,
&cfg.tcache_max);
}
emitter_kv(emitter, "nbins", "Number of bin size classes",
emitter_type_unsigned, &cfg.nbins);
emitter_kv(emitter, "nhbins", "Number of thread-cache bin size classes",
emitter_type_unsigned, &cfg.nhbins);
/*
* Per-size-class geometry is omitted from table output. It is present
* in JSON whenever general output is enabled; when omitted, it is not
* gathered either.
*/
if (!omit_size_class_meta) {
stats_general_bin_meta_print(emitter, cfg.nbins);
}
emitter_kv(emitter, "nlextents", "Number of large size classes",
emitter_type_unsigned, &cfg.nlextents);
if (!omit_size_class_meta) {
stats_general_lextent_meta_print(emitter, cfg.nlextents);
}
emitter_json_object_end(emitter); /* Close "arenas" */
}
JEMALLOC_COLD
static void
stats_general_print(emitter_t *emitter, bool omit_size_class_meta) {
stats_general_version(emitter);
stats_general_config(emitter);
stats_general_system(emitter);
stats_general_opts(emitter);
stats_general_prof(emitter);
stats_general_arenas_print(emitter, omit_size_class_meta);
}
static void
stats_emit_global(emitter_t *emitter, const stats_global_t *g) {
/* Generic global stats. */
emitter_json_kv(emitter, "allocated", emitter_type_size, &g->allocated);
emitter_json_kv(emitter, "active", emitter_type_size, &g->active);
emitter_json_kv(emitter, "metadata", emitter_type_size, &g->metadata);
emitter_json_kv(
emitter, "metadata_edata", emitter_type_size, &g->metadata_edata);
emitter_json_kv(
emitter, "metadata_rtree", emitter_type_size, &g->metadata_rtree);
emitter_json_kv(
emitter, "metadata_thp", emitter_type_size, &g->metadata_thp);
emitter_json_kv(emitter, "resident", emitter_type_size, &g->resident);
emitter_json_kv(emitter, "mapped", emitter_type_size, &g->mapped);
emitter_json_kv(emitter, "retained", emitter_type_size, &g->retained);
emitter_json_kv(emitter, "pinned", emitter_type_size, &g->pinned);
emitter_json_kv(
emitter, "zero_reallocs", emitter_type_size, &g->zero_reallocs);
emitter_table_printf(emitter,
"Allocated: %zu, active: %zu, "
"metadata: %zu (n_thp %zu, edata %zu, rtree %zu), resident: %zu, "
"mapped: %zu, retained: %zu, pinned: %zu\n",
g->allocated, g->active, g->metadata, g->metadata_thp,
g->metadata_edata, g->metadata_rtree, g->resident, g->mapped,
g->retained, g->pinned);
/* Strange behaviors */
emitter_table_printf(emitter,
"Count of realloc(non-null-ptr, 0) calls: %zu\n", g->zero_reallocs);
/* Background thread stats. */
emitter_json_object_kv_begin(emitter, "background_thread");
emitter_json_kv(emitter, "num_threads", emitter_type_size,
&g->num_background_threads);
emitter_json_kv(emitter, "num_runs", emitter_type_uint64,
&g->background_thread_num_runs);
emitter_json_kv(emitter, "run_interval", emitter_type_uint64,
&g->background_thread_run_interval);
emitter_json_object_end(emitter); /* Close "background_thread". */
emitter_table_printf(emitter,
"Background threads: %zu, "
"num_runs: %" FMTu64 ", run_interval: %" FMTu64 " ns\n",
g->num_background_threads, g->background_thread_num_runs,
g->background_thread_run_interval);
}
static void
stats_global_mutexes_print(emitter_t *emitter) {
/* Counters are gathered into emitter columns; see the arena variant. */
emitter_row_t row;
emitter_col_t name;
emitter_col_t col64[mutex_prof_num_uint64_t_counters];
emitter_col_t col32[mutex_prof_num_uint32_t_counters];
uint64_t uptime;
emitter_row_init(&row);
mutex_stats_init_cols(&row, "", &name, col64, col32);
emitter_table_row(emitter, &row);
emitter_json_object_kv_begin(emitter, "mutexes");
CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t);
size_t stats_mutexes_mib[CTL_MAX_DEPTH];
CTL_LEAF_PREPARE(stats_mutexes_mib, 0, "stats.mutexes");
for (int i = 0; i < mutex_prof_num_global_mutexes; i++) {
mutex_stats_read(stats_mutexes_mib, 2,
global_mutex_names[i], &name, col64, col32, uptime);
emitter_json_object_kv_begin(emitter, global_mutex_names[i]);
mutex_stats_emit(emitter, &row, col64, col32);
emitter_json_object_end(emitter);
}
emitter_json_object_end(emitter); /* Close "mutexes". */
}
static void
stats_print_globals(emitter_t *emitter, bool mutex) {
stats_global_t g;
stats_gather_global(&g);
emitter_json_object_kv_begin(emitter, "stats");
stats_emit_global(emitter, &g);
if (mutex) {
stats_global_mutexes_print(emitter);
}
emitter_json_object_end(emitter); /* Close "stats". */
}
static void
stats_print_one_arena(emitter_t *emitter, unsigned arena_ind,
const char *json_key, const char *table_header, bool bins, bool large,
bool mutex, bool extents, bool hpa) {
emitter_json_object_kv_begin(emitter, json_key);
emitter_table_printf(emitter, "%s", table_header);
stats_arena_print(emitter, arena_ind, bins, large, mutex, extents, hpa);
emitter_json_object_end(emitter);
}
JEMALLOC_COLD
static void
stats_print_runtime_stats(emitter_t *emitter, bool merged, bool destroyed,
bool unmerged, bool bins, bool large, bool mutex, bool extents, bool hpa) {
stats_print_globals(emitter, mutex);
if (!merged && !destroyed && !unmerged) {
return;
}
unsigned narenas;
CTL_GET("arenas.narenas", &narenas, unsigned);
VARIABLE_ARRAY_UNSAFE(bool, initialized, narenas);
bool destroyed_initialized;
unsigned ninitialized = stats_gather_arenas_initialized(narenas,
initialized, &destroyed_initialized);
emitter_json_object_kv_begin(emitter, "stats.arenas");
/* Merged stats. */
if (merged && (ninitialized > 1 || !unmerged)) {
stats_print_one_arena(emitter, MALLCTL_ARENAS_ALL, "merged",
"Merged arenas stats:\n", bins, large, mutex, extents, hpa);
}
/* Destroyed stats. */
if (destroyed_initialized && destroyed) {
stats_print_one_arena(emitter, MALLCTL_ARENAS_DESTROYED,
"destroyed", "Destroyed arenas stats:\n", bins, large, mutex,
extents, hpa);
}
/* Unmerged stats. */
if (unmerged) {
for (unsigned i = 0; i < narenas; i++) {
if (!initialized[i]) {
continue;
}
char arena_ind_str[20];
malloc_snprintf(arena_ind_str, sizeof(arena_ind_str),
"%u", i);
char header[32];
malloc_snprintf(header, sizeof(header),
"arenas[%s]:\n", arena_ind_str);
stats_print_one_arena(emitter, i, arena_ind_str, header,
bins, large, mutex, extents, hpa);
}
}
emitter_json_object_end(emitter); /* Close "stats.arenas". */
}
static uint64_t
stats_interval_new_event_wait(tsd_t *tsd) {
return stats_interval_accum_batch;
}
static uint64_t
stats_interval_postponed_event_wait(tsd_t *tsd) {
return TE_MIN_START_WAIT;
}
static void
stats_interval_event_handler(tsd_t *tsd) {
uint64_t last_event = thread_allocated_last_event_get(tsd);
uint64_t last_sample_event = tsd_stats_interval_last_event_get(tsd);
tsd_stats_interval_last_event_set(tsd, last_event);
uint64_t elapsed = last_event - last_sample_event;
assert(elapsed > 0 && elapsed != TE_INVALID_ELAPSED);
if (counter_accum(
tsd_tsdn(tsd), &stats_interval_accumulated, elapsed)) {
je_malloc_stats_print(NULL, NULL, opt_stats_interval_opts);
}
}
static te_enabled_t
stats_interval_enabled(void) {
return opt_stats_interval >= 0 ? te_enabled_yes : te_enabled_no;
}
te_base_cb_t stats_interval_te_handler = {
.enabled = &stats_interval_enabled,
.new_event_wait = &stats_interval_new_event_wait,
.postponed_event_wait = &stats_interval_postponed_event_wait,
.event_handler = &stats_interval_event_handler,
};
bool
stats_boot(void) {
uint64_t stats_interval;
if (opt_stats_interval < 0) {
assert(opt_stats_interval == -1);
stats_interval = 0;
stats_interval_accum_batch = 0;
} else {
/* See comments in stats.h */
stats_interval = (opt_stats_interval > 0) ? opt_stats_interval
: 1;
uint64_t batch = stats_interval
>> STATS_INTERVAL_ACCUM_LG_BATCH_SIZE;
if (batch > STATS_INTERVAL_ACCUM_BATCH_MAX) {
batch = STATS_INTERVAL_ACCUM_BATCH_MAX;
} else if (batch == 0) {
batch = 1;
}
stats_interval_accum_batch = batch;
}
return counter_accum_init(&stats_interval_accumulated, stats_interval);
}
void
stats_prefork(tsdn_t *tsdn) {
counter_prefork(tsdn, &stats_interval_accumulated);
}
void
stats_postfork_parent(tsdn_t *tsdn) {
counter_postfork_parent(tsdn, &stats_interval_accumulated);
}
void
stats_postfork_child(tsdn_t *tsdn) {
counter_postfork_child(tsdn, &stats_interval_accumulated);
}
Reference in a new issue View git blame Copy permalink