#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"

#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/ctl.h"
#include "jemalloc/internal/ctl_arena.h"
#include "jemalloc/internal/ctl_background_thread.h"
#include "jemalloc/internal/ctl_config.h"
#include "jemalloc/internal/ctl_mallctl.h"
#include "jemalloc/internal/ctl_opt.h"
#include "jemalloc/internal/ctl_prof.h"
#include "jemalloc/internal/ctl_stats.h"
#include "jemalloc/internal/ctl_tcache.h"
#include "jemalloc/internal/ctl_thread.h"
#include "jemalloc/internal/ctl_utilization.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/util.h"

/******************************************************************************/
/* Data. */

/*
 * ctl_mtx protects ctl stats state and arena ctl state.
 */
static malloc_mutex_t ctl_mtx;
static bool           ctl_initialized;

/******************************************************************************/
/* Helpers for named and indexed nodes. */

static const ctl_named_node_t *
ctl_named_node(const ctl_node_t *node) {
	return ((node->named) ? (const ctl_named_node_t *)node : NULL);
}

static const ctl_named_node_t *
ctl_named_children(const ctl_named_node_t *node, size_t index) {
	const ctl_named_node_t *children = ctl_named_node(node->children);

	return (children ? &children[index] : NULL);
}

static const ctl_indexed_node_t *
ctl_indexed_node(const ctl_node_t *node) {
	return (!node->named ? (const ctl_indexed_node_t *)node : NULL);
}

/******************************************************************************/
/* Function prototypes for non-inline static functions. */

#define CTL_PROTO(n)                                                           \
	static int n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,       \
	    void *oldp, size_t *oldlenp, void *newp, size_t newlen);

#define INDEX_PROTO(n)                                                         \
	static const ctl_named_node_t *n##_index(                              \
	    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i);

CTL_PROTO(version)
CTL_PROTO(epoch)
INDEX_PROTO(arena_i)
INDEX_PROTO(arenas_bin_i)
INDEX_PROTO(arenas_lextent_i)
INDEX_PROTO(stats_arenas_i_bins_j)
INDEX_PROTO(stats_arenas_i_lextents_j)
INDEX_PROTO(stats_arenas_i_extents_j)
INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j)
INDEX_PROTO(stats_arenas_i_hpa_shard_alloc_j)
INDEX_PROTO(stats_arenas_i)
INDEX_PROTO(prof_stats_bins_i)
INDEX_PROTO(prof_stats_lextents_i)
/******************************************************************************/
/*
 * mallctl tree.
 *
 * Handler implementations live in ctl_* namespace modules. Indexed-node
 * callbacks stay here because they navigate this tree and return tree nodes.
 */

#define NAME(n) {true}, n
#define CHILD(t, c)                                                            \
	sizeof(c##_node) / sizeof(ctl_##t##_node_t), (ctl_node_t *)c##_node,   \
	    NULL
#define CTL(c) 0, NULL, c##_ctl
#ifdef JEMALLOC_NO_PRIVATE_NAMESPACE
#define CTL_EXTERNAL(c) 0, NULL, c##_ctl
#else
#define CTL_EXTERNAL(c) 0, NULL, JEMALLOC_N(c##_ctl)
#endif

/*
 * Only handles internal indexed nodes, since there are currently no external
 * ones.
 */
#define INDEX(i) {false}, i##_index

static const ctl_named_node_t thread_tcache_ncached_max_node[] = {
    {NAME("read_sizeclass"), CTL_EXTERNAL(thread_tcache_ncached_max_read_sizeclass)},
    {NAME("write"), CTL_EXTERNAL(thread_tcache_ncached_max_write)}};

static const ctl_named_node_t thread_tcache_node[] = {
    {NAME("enabled"), CTL_EXTERNAL(thread_tcache_enabled)},
    {NAME("max"), CTL_EXTERNAL(thread_tcache_max)},
    {NAME("flush"), CTL_EXTERNAL(thread_tcache_flush)},
    {NAME("ncached_max"), CHILD(named, thread_tcache_ncached_max)}};

static const ctl_named_node_t thread_peak_node[] = {
    {NAME("read"), CTL_EXTERNAL(thread_peak_read)},
    {NAME("reset"), CTL_EXTERNAL(thread_peak_reset)},
};

static const ctl_named_node_t thread_prof_node[] = {
    {NAME("name"), CTL_EXTERNAL(thread_prof_name)},
    {NAME("active"), CTL_EXTERNAL(thread_prof_active)}};

static const ctl_named_node_t thread_node[] = {
    {NAME("arena"), CTL_EXTERNAL(thread_arena)},
    {NAME("allocated"), CTL_EXTERNAL(thread_allocated)},
    {NAME("allocatedp"), CTL_EXTERNAL(thread_allocatedp)},
    {NAME("deallocated"), CTL_EXTERNAL(thread_deallocated)},
    {NAME("deallocatedp"), CTL_EXTERNAL(thread_deallocatedp)},
    {NAME("tcache"), CHILD(named, thread_tcache)},
    {NAME("peak"), CHILD(named, thread_peak)},
    {NAME("prof"), CHILD(named, thread_prof)},
    {NAME("idle"), CTL_EXTERNAL(thread_idle)}};

static const ctl_named_node_t config_node[] = {
    {NAME("cache_oblivious"), CTL_EXTERNAL(config_cache_oblivious)},
    {NAME("debug"), CTL_EXTERNAL(config_debug)},
    {NAME("fill"), CTL_EXTERNAL(config_fill)},
    {NAME("lazy_lock"), CTL_EXTERNAL(config_lazy_lock)},
    {NAME("malloc_conf"), CTL_EXTERNAL(config_malloc_conf)},
    {NAME("opt_safety_checks"), CTL_EXTERNAL(config_opt_safety_checks)},
    {NAME("prof"), CTL_EXTERNAL(config_prof)},
    {NAME("prof_libgcc"), CTL_EXTERNAL(config_prof_libgcc)},
    {NAME("prof_libunwind"), CTL_EXTERNAL(config_prof_libunwind)},
    {NAME("prof_frameptr"), CTL_EXTERNAL(config_prof_frameptr)},
    {NAME("stats"), CTL_EXTERNAL(config_stats)},
    {NAME("utrace"), CTL_EXTERNAL(config_utrace)},
    {NAME("xmalloc"), CTL_EXTERNAL(config_xmalloc)}};

static const ctl_named_node_t opt_malloc_conf_node[] = {
    {NAME("symlink"), CTL_EXTERNAL(opt_malloc_conf_symlink)},
    {NAME("env_var"), CTL_EXTERNAL(opt_malloc_conf_env_var)},
    {NAME("global_var"), CTL_EXTERNAL(opt_malloc_conf_global_var)}};

static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL_EXTERNAL(opt_abort)},
    {NAME("abort_conf"), CTL_EXTERNAL(opt_abort_conf)},
    {NAME("cache_oblivious"), CTL_EXTERNAL(opt_cache_oblivious)},
    {NAME("trust_madvise"), CTL_EXTERNAL(opt_trust_madvise)},
    {NAME("experimental_hpa_start_huge_if_thp_always"),
        CTL_EXTERNAL(opt_experimental_hpa_start_huge_if_thp_always)},
    {NAME("experimental_hpa_enforce_hugify"),
        CTL_EXTERNAL(opt_experimental_hpa_enforce_hugify)},
    {NAME("confirm_conf"), CTL_EXTERNAL(opt_confirm_conf)}, {NAME("hpa"), CTL_EXTERNAL(opt_hpa)},
    {NAME("hpa_slab_max_alloc"), CTL_EXTERNAL(opt_hpa_slab_max_alloc)},
    {NAME("hpa_hugification_threshold"), CTL_EXTERNAL(opt_hpa_hugification_threshold)},
    {NAME("hpa_hugify_delay_ms"), CTL_EXTERNAL(opt_hpa_hugify_delay_ms)},
    {NAME("hpa_hugify_sync"), CTL_EXTERNAL(opt_hpa_hugify_sync)},
    {NAME("hpa_min_purge_interval_ms"), CTL_EXTERNAL(opt_hpa_min_purge_interval_ms)},
    {NAME("experimental_hpa_max_purge_nhp"),
        CTL_EXTERNAL(opt_experimental_hpa_max_purge_nhp)},
    {NAME("hpa_purge_threshold"), CTL_EXTERNAL(opt_hpa_purge_threshold)},
    {NAME("hpa_min_purge_delay_ms"), CTL_EXTERNAL(opt_hpa_min_purge_delay_ms)},
    {NAME("hpa_hugify_style"), CTL_EXTERNAL(opt_hpa_hugify_style)},
    {NAME("hpa_dirty_mult"), CTL_EXTERNAL(opt_hpa_dirty_mult)},
    {NAME("hpa_sec_nshards"), CTL_EXTERNAL(opt_hpa_sec_nshards)},
    {NAME("hpa_sec_max_alloc"), CTL_EXTERNAL(opt_hpa_sec_max_alloc)},
    {NAME("hpa_sec_max_bytes"), CTL_EXTERNAL(opt_hpa_sec_max_bytes)},
    {NAME("huge_arena_pac_thp"), CTL_EXTERNAL(opt_huge_arena_pac_thp)},
    {NAME("metadata_thp"), CTL_EXTERNAL(opt_metadata_thp)},
    {NAME("retain"), CTL_EXTERNAL(opt_retain)}, {NAME("dss"), CTL_EXTERNAL(opt_dss)},
    {NAME("narenas"), CTL_EXTERNAL(opt_narenas)},
    {NAME("percpu_arena"), CTL_EXTERNAL(opt_percpu_arena)},
    {NAME("oversize_threshold"), CTL_EXTERNAL(opt_oversize_threshold)},
    {NAME("mutex_max_spin"), CTL_EXTERNAL(opt_mutex_max_spin)},
    {NAME("background_thread"), CTL_EXTERNAL(opt_background_thread)},
    {NAME("max_background_threads"), CTL_EXTERNAL(opt_max_background_threads)},
    {NAME("dirty_decay_ms"), CTL_EXTERNAL(opt_dirty_decay_ms)},
    {NAME("muzzy_decay_ms"), CTL_EXTERNAL(opt_muzzy_decay_ms)},
    {NAME("stats_print"), CTL_EXTERNAL(opt_stats_print)},
    {NAME("stats_print_opts"), CTL_EXTERNAL(opt_stats_print_opts)},
    {NAME("stats_interval"), CTL_EXTERNAL(opt_stats_interval)},
    {NAME("stats_interval_opts"), CTL_EXTERNAL(opt_stats_interval_opts)},
    {NAME("junk"), CTL_EXTERNAL(opt_junk)}, {NAME("zero"), CTL_EXTERNAL(opt_zero)},
    {NAME("utrace"), CTL_EXTERNAL(opt_utrace)}, {NAME("xmalloc"), CTL_EXTERNAL(opt_xmalloc)},
    {NAME("experimental_infallible_new"), CTL_EXTERNAL(opt_experimental_infallible_new)},
    {NAME("experimental_tcache_gc"), CTL_EXTERNAL(opt_experimental_tcache_gc)},
    {NAME("tcache"), CTL_EXTERNAL(opt_tcache)},
    {NAME("tcache_max"), CTL_EXTERNAL(opt_tcache_max)},
    {NAME("tcache_nslots_small_min"), CTL_EXTERNAL(opt_tcache_nslots_small_min)},
    {NAME("tcache_nslots_small_max"), CTL_EXTERNAL(opt_tcache_nslots_small_max)},
    {NAME("tcache_nslots_large"), CTL_EXTERNAL(opt_tcache_nslots_large)},
    {NAME("lg_tcache_nslots_mul"), CTL_EXTERNAL(opt_lg_tcache_nslots_mul)},
    {NAME("tcache_gc_incr_bytes"), CTL_EXTERNAL(opt_tcache_gc_incr_bytes)},
    {NAME("tcache_gc_delay_bytes"), CTL_EXTERNAL(opt_tcache_gc_delay_bytes)},
    {NAME("lg_tcache_flush_small_div"), CTL_EXTERNAL(opt_lg_tcache_flush_small_div)},
    {NAME("lg_tcache_flush_large_div"), CTL_EXTERNAL(opt_lg_tcache_flush_large_div)},
    {NAME("thp"), CTL_EXTERNAL(opt_thp)},
    {NAME("lg_extent_max_active_fit"), CTL_EXTERNAL(opt_lg_extent_max_active_fit)},
    {NAME("prof"), CTL_EXTERNAL(opt_prof)}, {NAME("prof_prefix"), CTL_EXTERNAL(opt_prof_prefix)},
    {NAME("prof_active"), CTL_EXTERNAL(opt_prof_active)},
    {NAME("prof_thread_active_init"), CTL_EXTERNAL(opt_prof_thread_active_init)},
    {NAME("prof_bt_max"), CTL_EXTERNAL(opt_prof_bt_max)},
    {NAME("lg_prof_sample"), CTL_EXTERNAL(opt_lg_prof_sample)},
    {NAME("lg_prof_interval"), CTL_EXTERNAL(opt_lg_prof_interval)},
    {NAME("prof_gdump"), CTL_EXTERNAL(opt_prof_gdump)},
    {NAME("prof_final"), CTL_EXTERNAL(opt_prof_final)},
    {NAME("prof_leak"), CTL_EXTERNAL(opt_prof_leak)},
    {NAME("prof_leak_error"), CTL_EXTERNAL(opt_prof_leak_error)},
    {NAME("prof_accum"), CTL_EXTERNAL(opt_prof_accum)},
    {NAME("prof_pid_namespace"), CTL_EXTERNAL(opt_prof_pid_namespace)},
    {NAME("prof_recent_alloc_max"), CTL_EXTERNAL(opt_prof_recent_alloc_max)},
    {NAME("prof_stats"), CTL_EXTERNAL(opt_prof_stats)},
    {NAME("prof_sys_thread_name"), CTL_EXTERNAL(opt_prof_sys_thread_name)},
    {NAME("prof_time_resolution"), CTL_EXTERNAL(opt_prof_time_res)},
    {NAME("lg_san_uaf_align"), CTL_EXTERNAL(opt_lg_san_uaf_align)},
    {NAME("zero_realloc"), CTL_EXTERNAL(opt_zero_realloc)},
    {NAME("debug_double_free_max_scan"), CTL_EXTERNAL(opt_debug_double_free_max_scan)},
    {NAME("disable_large_size_classes"), CTL_EXTERNAL(opt_disable_large_size_classes)},
    {NAME("process_madvise_max_batch"), CTL_EXTERNAL(opt_process_madvise_max_batch)},
    {NAME("malloc_conf"), CHILD(named, opt_malloc_conf)}};

static const ctl_named_node_t tcache_node[] = {
    {NAME("create"), CTL_EXTERNAL(tcache_create)},
    {NAME("flush"), CTL_EXTERNAL(tcache_flush)},
    {NAME("destroy"), CTL_EXTERNAL(tcache_destroy)}};

static const ctl_named_node_t arena_i_node[] = {
    {NAME("initialized"), CTL_EXTERNAL(arena_i_initialized)},
    {NAME("decay"), CTL_EXTERNAL(arena_i_decay)},
    {NAME("purge"), CTL_EXTERNAL(arena_i_purge)},
    {NAME("reset"), CTL_EXTERNAL(arena_i_reset)},
    {NAME("destroy"), CTL_EXTERNAL(arena_i_destroy)},
    {NAME("dss"), CTL_EXTERNAL(arena_i_dss)},
    /*
	 * Undocumented for now, since we anticipate an arena API in flux after
	 * we cut the last 5-series release.
	 */
    {NAME("oversize_threshold"), CTL_EXTERNAL(arena_i_oversize_threshold)},
    {NAME("dirty_decay_ms"), CTL_EXTERNAL(arena_i_dirty_decay_ms)},
    {NAME("muzzy_decay_ms"), CTL_EXTERNAL(arena_i_muzzy_decay_ms)},
    {NAME("extent_hooks"), CTL_EXTERNAL(arena_i_extent_hooks)},
    {NAME("retain_grow_limit"), CTL_EXTERNAL(arena_i_retain_grow_limit)},
    {NAME("name"), CTL_EXTERNAL(arena_i_name)}};
static const ctl_named_node_t super_arena_i_node[] = {
    {NAME(""), CHILD(named, arena_i)}};

static const ctl_indexed_node_t arena_node[] = {{INDEX(arena_i)}};

static const ctl_named_node_t arenas_bin_i_node[] = {
    {NAME("size"), CTL_EXTERNAL(arenas_bin_i_size)},
    {NAME("nregs"), CTL_EXTERNAL(arenas_bin_i_nregs)},
    {NAME("slab_size"), CTL_EXTERNAL(arenas_bin_i_slab_size)},
    {NAME("nshards"), CTL_EXTERNAL(arenas_bin_i_nshards)}};
static const ctl_named_node_t super_arenas_bin_i_node[] = {
    {NAME(""), CHILD(named, arenas_bin_i)}};

static const ctl_indexed_node_t arenas_bin_node[] = {{INDEX(arenas_bin_i)}};

static const ctl_named_node_t arenas_lextent_i_node[] = {
    {NAME("size"), CTL_EXTERNAL(arenas_lextent_i_size)}};
static const ctl_named_node_t super_arenas_lextent_i_node[] = {
    {NAME(""), CHILD(named, arenas_lextent_i)}};

static const ctl_indexed_node_t arenas_lextent_node[] = {
    {INDEX(arenas_lextent_i)}};

static const ctl_named_node_t arenas_node[] = {
    {NAME("narenas"), CTL_EXTERNAL(arenas_narenas)},
    {NAME("dirty_decay_ms"), CTL_EXTERNAL(arenas_dirty_decay_ms)},
    {NAME("muzzy_decay_ms"), CTL_EXTERNAL(arenas_muzzy_decay_ms)},
    {NAME("quantum"), CTL_EXTERNAL(arenas_quantum)},
    {NAME("page"), CTL_EXTERNAL(arenas_page)},
    {NAME("hugepage"), CTL_EXTERNAL(arenas_hugepage)},
    {NAME("tcache_max"), CTL_EXTERNAL(arenas_tcache_max)},
    {NAME("nbins"), CTL_EXTERNAL(arenas_nbins)},
    {NAME("nhbins"), CTL_EXTERNAL(arenas_nhbins)},
    {NAME("bin"), CHILD(indexed, arenas_bin)},
    {NAME("nlextents"), CTL_EXTERNAL(arenas_nlextents)},
    {NAME("lextent"), CHILD(indexed, arenas_lextent)},
    {NAME("create"), CTL_EXTERNAL(arenas_create)},
    {NAME("lookup"), CTL_EXTERNAL(arenas_lookup)}};

static const ctl_named_node_t prof_stats_bins_i_node[] = {
    {NAME("live"), CTL_EXTERNAL(prof_stats_bins_i_live)},
    {NAME("accum"), CTL_EXTERNAL(prof_stats_bins_i_accum)}};

static const ctl_named_node_t super_prof_stats_bins_i_node[] = {
    {NAME(""), CHILD(named, prof_stats_bins_i)}};

static const ctl_indexed_node_t prof_stats_bins_node[] = {
    {INDEX(prof_stats_bins_i)}};

static const ctl_named_node_t prof_stats_lextents_i_node[] = {
    {NAME("live"), CTL_EXTERNAL(prof_stats_lextents_i_live)},
    {NAME("accum"), CTL_EXTERNAL(prof_stats_lextents_i_accum)}};

static const ctl_named_node_t super_prof_stats_lextents_i_node[] = {
    {NAME(""), CHILD(named, prof_stats_lextents_i)}};

static const ctl_indexed_node_t prof_stats_lextents_node[] = {
    {INDEX(prof_stats_lextents_i)}};

static const ctl_named_node_t prof_stats_node[] = {
    {NAME("bins"), CHILD(indexed, prof_stats_bins)},
    {NAME("lextents"), CHILD(indexed, prof_stats_lextents)},
};

static const ctl_named_node_t prof_node[] = {
    {NAME("thread_active_init"), CTL_EXTERNAL(prof_thread_active_init)},
    {NAME("active"), CTL_EXTERNAL(prof_active)}, {NAME("dump"), CTL_EXTERNAL(prof_dump)},
    {NAME("gdump"), CTL_EXTERNAL(prof_gdump)}, {NAME("prefix"), CTL_EXTERNAL(prof_prefix)},
    {NAME("reset"), CTL_EXTERNAL(prof_reset)}, {NAME("interval"), CTL_EXTERNAL(prof_interval)},
    {NAME("lg_sample"), CTL_EXTERNAL(lg_prof_sample)},
    {NAME("log_start"), CTL_EXTERNAL(prof_log_start)},
    {NAME("log_stop"), CTL_EXTERNAL(prof_log_stop)},
    {NAME("stats"), CHILD(named, prof_stats)}};

static const ctl_named_node_t stats_arenas_i_small_node[] = {
    {NAME("allocated"), CTL_EXTERNAL(stats_arenas_i_small_allocated)},
    {NAME("nmalloc"), CTL_EXTERNAL(stats_arenas_i_small_nmalloc)},
    {NAME("ndalloc"), CTL_EXTERNAL(stats_arenas_i_small_ndalloc)},
    {NAME("nrequests"), CTL_EXTERNAL(stats_arenas_i_small_nrequests)},
    {NAME("nfills"), CTL_EXTERNAL(stats_arenas_i_small_nfills)},
    {NAME("nflushes"), CTL_EXTERNAL(stats_arenas_i_small_nflushes)}};

static const ctl_named_node_t stats_arenas_i_large_node[] = {
    {NAME("allocated"), CTL_EXTERNAL(stats_arenas_i_large_allocated)},
    {NAME("nmalloc"), CTL_EXTERNAL(stats_arenas_i_large_nmalloc)},
    {NAME("ndalloc"), CTL_EXTERNAL(stats_arenas_i_large_ndalloc)},
    {NAME("nrequests"), CTL_EXTERNAL(stats_arenas_i_large_nrequests)},
    {NAME("nfills"), CTL_EXTERNAL(stats_arenas_i_large_nfills)},
    {NAME("nflushes"), CTL_EXTERNAL(stats_arenas_i_large_nflushes)}};

#define MUTEX_PROF_DATA_NODE(prefix)                                                                          \
	static const ctl_named_node_t stats_##prefix##_node[] = {                                             \
	    {NAME("num_ops"), CTL_EXTERNAL(stats_##prefix##_num_ops)},                                                 \
	    {NAME("num_wait"), CTL_EXTERNAL(stats_##prefix##_num_wait)},                                               \
	    {NAME("num_spin_acq"), CTL_EXTERNAL(stats_##prefix##_num_spin_acq)},                                       \
	    {NAME("num_owner_switch"),                                                                        \
	        CTL_EXTERNAL(stats_##prefix##_num_owner_switch)},                                                      \
	    {NAME("total_wait_time"), CTL_EXTERNAL(stats_##prefix##_total_wait_time)},                                 \
	    {NAME("max_wait_time"), CTL_EXTERNAL(stats_##prefix##_max_wait_time)},                                     \
	    {NAME("max_num_thds"),                                                                            \
	        CTL_EXTERNAL(stats_##prefix##_max_num_thds)} /* Note that # of current waiting thread not provided. */ \
	};

MUTEX_PROF_DATA_NODE(arenas_i_bins_j_mutex)

static const ctl_named_node_t stats_arenas_i_bins_j_node[] = {
    {NAME("nmalloc"), CTL_EXTERNAL(stats_arenas_i_bins_j_nmalloc)},
    {NAME("ndalloc"), CTL_EXTERNAL(stats_arenas_i_bins_j_ndalloc)},
    {NAME("nrequests"), CTL_EXTERNAL(stats_arenas_i_bins_j_nrequests)},
    {NAME("curregs"), CTL_EXTERNAL(stats_arenas_i_bins_j_curregs)},
    {NAME("nfills"), CTL_EXTERNAL(stats_arenas_i_bins_j_nfills)},
    {NAME("nflushes"), CTL_EXTERNAL(stats_arenas_i_bins_j_nflushes)},
    {NAME("nslabs"), CTL_EXTERNAL(stats_arenas_i_bins_j_nslabs)},
    {NAME("nreslabs"), CTL_EXTERNAL(stats_arenas_i_bins_j_nreslabs)},
    {NAME("curslabs"), CTL_EXTERNAL(stats_arenas_i_bins_j_curslabs)},
    {NAME("nonfull_slabs"), CTL_EXTERNAL(stats_arenas_i_bins_j_nonfull_slabs)},
    {NAME("mutex"), CHILD(named, stats_arenas_i_bins_j_mutex)}};

static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = {
    {NAME(""), CHILD(named, stats_arenas_i_bins_j)}};

static const ctl_indexed_node_t stats_arenas_i_bins_node[] = {
    {INDEX(stats_arenas_i_bins_j)}};

static const ctl_named_node_t stats_arenas_i_lextents_j_node[] = {
    {NAME("nmalloc"), CTL_EXTERNAL(stats_arenas_i_lextents_j_nmalloc)},
    {NAME("ndalloc"), CTL_EXTERNAL(stats_arenas_i_lextents_j_ndalloc)},
    {NAME("nrequests"), CTL_EXTERNAL(stats_arenas_i_lextents_j_nrequests)},
    {NAME("curlextents"), CTL_EXTERNAL(stats_arenas_i_lextents_j_curlextents)}};
static const ctl_named_node_t super_stats_arenas_i_lextents_j_node[] = {
    {NAME(""), CHILD(named, stats_arenas_i_lextents_j)}};

static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = {
    {INDEX(stats_arenas_i_lextents_j)}};

static const ctl_named_node_t stats_arenas_i_extents_j_node[] = {
    {NAME("ndirty"), CTL_EXTERNAL(stats_arenas_i_extents_j_ndirty)},
    {NAME("nmuzzy"), CTL_EXTERNAL(stats_arenas_i_extents_j_nmuzzy)},
    {NAME("nretained"), CTL_EXTERNAL(stats_arenas_i_extents_j_nretained)},
    {NAME("npinned"), CTL_EXTERNAL(stats_arenas_i_extents_j_npinned)},
    {NAME("dirty_bytes"), CTL_EXTERNAL(stats_arenas_i_extents_j_dirty_bytes)},
    {NAME("muzzy_bytes"), CTL_EXTERNAL(stats_arenas_i_extents_j_muzzy_bytes)},
    {NAME("retained_bytes"), CTL_EXTERNAL(stats_arenas_i_extents_j_retained_bytes)},
    {NAME("pinned_bytes"), CTL_EXTERNAL(stats_arenas_i_extents_j_pinned_bytes)}};

static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = {
    {NAME(""), CHILD(named, stats_arenas_i_extents_j)}};

static const ctl_indexed_node_t stats_arenas_i_extents_node[] = {
    {INDEX(stats_arenas_i_extents_j)}};

#define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx)
MUTEX_PROF_ARENA_MUTEXES
#undef OP

static const ctl_named_node_t stats_arenas_i_mutexes_node[] = {
#define OP(mtx) {NAME(#mtx), CHILD(named, stats_arenas_i_mutexes_##mtx)},
    MUTEX_PROF_ARENA_MUTEXES
#undef OP
};

static const ctl_named_node_t stats_arenas_i_hpa_shard_slabs_node[] = {
    {NAME("npageslabs_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_slabs_npageslabs_nonhuge)},
    {NAME("npageslabs_huge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_slabs_npageslabs_huge)},
    {NAME("nactive_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_slabs_nactive_nonhuge)},
    {NAME("nactive_huge"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_slabs_nactive_huge)},
    {NAME("ndirty_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_slabs_ndirty_nonhuge)},
    {NAME("ndirty_huge"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_slabs_ndirty_huge)}};

static const ctl_named_node_t stats_arenas_i_hpa_shard_full_slabs_node[] = {
    {NAME("npageslabs_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge)},
    {NAME("npageslabs_huge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_full_slabs_npageslabs_huge)},
    {NAME("nactive_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_full_slabs_nactive_nonhuge)},
    {NAME("nactive_huge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_full_slabs_nactive_huge)},
    {NAME("ndirty_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_full_slabs_ndirty_nonhuge)},
    {NAME("ndirty_huge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_full_slabs_ndirty_huge)}};

static const ctl_named_node_t stats_arenas_i_hpa_shard_empty_slabs_node[] = {
    {NAME("npageslabs_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_nonhuge)},
    {NAME("npageslabs_huge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_empty_slabs_npageslabs_huge)},
    {NAME("nactive_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_empty_slabs_nactive_nonhuge)},
    {NAME("nactive_huge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_empty_slabs_nactive_huge)},
    {NAME("ndirty_nonhuge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_nonhuge)},
    {NAME("ndirty_huge"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_empty_slabs_ndirty_huge)}};

static const ctl_named_node_t stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] =
    {{NAME("npageslabs_nonhuge"),
         CTL_EXTERNAL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_nonhuge)},
        {NAME("npageslabs_huge"),
            CTL_EXTERNAL(stats_arenas_i_hpa_shard_nonfull_slabs_j_npageslabs_huge)},
        {NAME("nactive_nonhuge"),
            CTL_EXTERNAL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_nonhuge)},
        {NAME("nactive_huge"),
            CTL_EXTERNAL(stats_arenas_i_hpa_shard_nonfull_slabs_j_nactive_huge)},
        {NAME("ndirty_nonhuge"),
            CTL_EXTERNAL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)},
        {NAME("ndirty_huge"),
            CTL_EXTERNAL(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)}};

static const ctl_named_node_t
    super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node[] = {
        {NAME(""), CHILD(named, stats_arenas_i_hpa_shard_nonfull_slabs_j)}};

static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] =
    {{INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}};

static const ctl_named_node_t stats_arenas_i_hpa_shard_alloc_j_node[] = {
    {NAME("min_extents"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_alloc_j_min_extents)},
    {NAME("max_extents"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_alloc_j_max_extents)},
    {NAME("extents"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_alloc_j_extents)},
    {NAME("ps"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_alloc_j_ps)},
    {NAME("pages_per_ps"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_alloc_j_pages_per_ps)},
    {NAME("extents_per_ps"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_alloc_j_extents_per_ps)},
    {NAME("total_elapsed_ns_per_ps"),
        CTL_EXTERNAL(stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps)}};

static const ctl_named_node_t super_stats_arenas_i_hpa_shard_alloc_j_node[] = {
    {NAME(""), CHILD(named, stats_arenas_i_hpa_shard_alloc_j)}};

static const ctl_indexed_node_t stats_arenas_i_hpa_shard_alloc_node[] = {
    {INDEX(stats_arenas_i_hpa_shard_alloc_j)}};

static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
    {NAME("npageslabs"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_npageslabs)},
    {NAME("nactive"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_nactive)},
    {NAME("ndirty"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_ndirty)},

    {NAME("slabs"), CHILD(named, stats_arenas_i_hpa_shard_slabs)},

    {NAME("npurge_passes"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_npurge_passes)},
    {NAME("npurges"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_npurges)},
    {NAME("nhugifies"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_nhugifies)},
    {NAME("nhugify_failures"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_nhugify_failures)},
    {NAME("ndehugifies"), CTL_EXTERNAL(stats_arenas_i_hpa_shard_ndehugifies)},

    {NAME("alloc"), CHILD(indexed, stats_arenas_i_hpa_shard_alloc)},

    {NAME("full_slabs"), CHILD(named, stats_arenas_i_hpa_shard_full_slabs)},
    {NAME("empty_slabs"), CHILD(named, stats_arenas_i_hpa_shard_empty_slabs)},
    {NAME("nonfull_slabs"),
        CHILD(indexed, stats_arenas_i_hpa_shard_nonfull_slabs)}};

static const ctl_named_node_t stats_arenas_i_node[] = {
    {NAME("nthreads"), CTL_EXTERNAL(stats_arenas_i_nthreads)},
    {NAME("uptime"), CTL_EXTERNAL(stats_arenas_i_uptime)},
    {NAME("dss"), CTL_EXTERNAL(stats_arenas_i_dss)},
    {NAME("dirty_decay_ms"), CTL_EXTERNAL(stats_arenas_i_dirty_decay_ms)},
    {NAME("muzzy_decay_ms"), CTL_EXTERNAL(stats_arenas_i_muzzy_decay_ms)},
    {NAME("pactive"), CTL_EXTERNAL(stats_arenas_i_pactive)},
    {NAME("pdirty"), CTL_EXTERNAL(stats_arenas_i_pdirty)},
    {NAME("pmuzzy"), CTL_EXTERNAL(stats_arenas_i_pmuzzy)},
    {NAME("mapped"), CTL_EXTERNAL(stats_arenas_i_mapped)},
    {NAME("retained"), CTL_EXTERNAL(stats_arenas_i_retained)},
    {NAME("pinned"), CTL_EXTERNAL(stats_arenas_i_pinned)},
    {NAME("extent_avail"), CTL_EXTERNAL(stats_arenas_i_extent_avail)},
    {NAME("dirty_npurge"), CTL_EXTERNAL(stats_arenas_i_dirty_npurge)},
    {NAME("dirty_nmadvise"), CTL_EXTERNAL(stats_arenas_i_dirty_nmadvise)},
    {NAME("dirty_purged"), CTL_EXTERNAL(stats_arenas_i_dirty_purged)},
    {NAME("muzzy_npurge"), CTL_EXTERNAL(stats_arenas_i_muzzy_npurge)},
    {NAME("muzzy_nmadvise"), CTL_EXTERNAL(stats_arenas_i_muzzy_nmadvise)},
    {NAME("muzzy_purged"), CTL_EXTERNAL(stats_arenas_i_muzzy_purged)},
    {NAME("base"), CTL_EXTERNAL(stats_arenas_i_base)},
    {NAME("internal"), CTL_EXTERNAL(stats_arenas_i_internal)},
    {NAME("metadata_edata"), CTL_EXTERNAL(stats_arenas_i_metadata_edata)},
    {NAME("metadata_rtree"), CTL_EXTERNAL(stats_arenas_i_metadata_rtree)},
    {NAME("metadata_thp"), CTL_EXTERNAL(stats_arenas_i_metadata_thp)},
    {NAME("tcache_bytes"), CTL_EXTERNAL(stats_arenas_i_tcache_bytes)},
    {NAME("tcache_stashed_bytes"), CTL_EXTERNAL(stats_arenas_i_tcache_stashed_bytes)},
    {NAME("resident"), CTL_EXTERNAL(stats_arenas_i_resident)},
    {NAME("abandoned_vm"), CTL_EXTERNAL(stats_arenas_i_abandoned_vm)},
    {NAME("hpa_sec_bytes"), CTL_EXTERNAL(stats_arenas_i_hpa_sec_bytes)},
    {NAME("hpa_sec_hits"), CTL_EXTERNAL(stats_arenas_i_hpa_sec_hits)},
    {NAME("hpa_sec_misses"), CTL_EXTERNAL(stats_arenas_i_hpa_sec_misses)},
    {NAME("hpa_sec_dalloc_noflush"),
        CTL_EXTERNAL(stats_arenas_i_hpa_sec_dalloc_noflush)},
    {NAME("hpa_sec_dalloc_flush"), CTL_EXTERNAL(stats_arenas_i_hpa_sec_dalloc_flush)},
    {NAME("hpa_sec_overfills"), CTL_EXTERNAL(stats_arenas_i_hpa_sec_overfills)},
    {NAME("small"), CHILD(named, stats_arenas_i_small)},
    {NAME("large"), CHILD(named, stats_arenas_i_large)},
    {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
    {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)},
    {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)},
    {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)},
    {NAME("hpa_shard"), CHILD(named, stats_arenas_i_hpa_shard)}};
static const ctl_named_node_t super_stats_arenas_i_node[] = {
    {NAME(""), CHILD(named, stats_arenas_i)}};

static const ctl_indexed_node_t stats_arenas_node[] = {{INDEX(stats_arenas_i)}};

static const ctl_named_node_t stats_background_thread_node[] = {
    {NAME("num_threads"), CTL_EXTERNAL(stats_background_thread_num_threads)},
    {NAME("num_runs"), CTL_EXTERNAL(stats_background_thread_num_runs)},
    {NAME("run_interval"), CTL_EXTERNAL(stats_background_thread_run_interval)}};

#define OP(mtx) MUTEX_PROF_DATA_NODE(mutexes_##mtx)
MUTEX_PROF_GLOBAL_MUTEXES
#undef OP

static const ctl_named_node_t stats_mutexes_node[] = {
#define OP(mtx) {NAME(#mtx), CHILD(named, stats_mutexes_##mtx)},
    MUTEX_PROF_GLOBAL_MUTEXES
#undef OP
    {NAME("reset"), CTL_EXTERNAL(stats_mutexes_reset)}};
#undef MUTEX_PROF_DATA_NODE

static const ctl_named_node_t approximate_stats_node[] = {
    {NAME("active"), CTL_EXTERNAL(approximate_stats_active)},
};

static const ctl_named_node_t stats_node[] = {
    {NAME("allocated"), CTL_EXTERNAL(stats_allocated)},
    {NAME("active"), CTL_EXTERNAL(stats_active)},
    {NAME("metadata"), CTL_EXTERNAL(stats_metadata)},
    {NAME("metadata_edata"), CTL_EXTERNAL(stats_metadata_edata)},
    {NAME("metadata_rtree"), CTL_EXTERNAL(stats_metadata_rtree)},
    {NAME("metadata_thp"), CTL_EXTERNAL(stats_metadata_thp)},
    {NAME("resident"), CTL_EXTERNAL(stats_resident)},
    {NAME("mapped"), CTL_EXTERNAL(stats_mapped)},
    {NAME("retained"), CTL_EXTERNAL(stats_retained)},
    {NAME("pinned"), CTL_EXTERNAL(stats_pinned)},
    {NAME("background_thread"), CHILD(named, stats_background_thread)},
    {NAME("mutexes"), CHILD(named, stats_mutexes)},
    {NAME("arenas"), CHILD(indexed, stats_arenas)},
    {NAME("zero_reallocs"), CTL_EXTERNAL(stats_zero_reallocs)},
};

static const ctl_named_node_t experimental_hooks_node[] = {
    {NAME("prof_backtrace"), CTL_EXTERNAL(experimental_hooks_prof_backtrace)},
    {NAME("prof_dump"), CTL_EXTERNAL(experimental_hooks_prof_dump)},
    {NAME("prof_sample"), CTL_EXTERNAL(experimental_hooks_prof_sample)},
    {NAME("prof_sample_free"), CTL_EXTERNAL(experimental_hooks_prof_sample_free)},
    {NAME("thread_event"), CTL_EXTERNAL(experimental_hooks_thread_event)},
};

static const ctl_named_node_t experimental_prof_recent_node[] = {
    {NAME("alloc_max"), CTL_EXTERNAL(experimental_prof_recent_alloc_max)},
    {NAME("alloc_dump"), CTL_EXTERNAL(experimental_prof_recent_alloc_dump)},
};

static const ctl_named_node_t experimental_utilization_node[] = {
    {NAME("batch_query"), CTL_EXTERNAL(experimental_utilization_batch_query)}};

static const ctl_named_node_t experimental_node[] = {
    {NAME("hooks"), CHILD(named, experimental_hooks)},
    {NAME("arenas_create_ext"), CTL_EXTERNAL(experimental_arenas_create_ext)},
    {NAME("prof_recent"), CHILD(named, experimental_prof_recent)},
    {NAME("utilization"), CHILD(named, experimental_utilization)}};

static const ctl_named_node_t root_node[] = {{NAME("version"), CTL(version)},
    {NAME("epoch"), CTL(epoch)},
    {NAME("background_thread"), CTL_EXTERNAL(background_thread)},
    {NAME("max_background_threads"), CTL_EXTERNAL(max_background_threads)},
    {NAME("thread"), CHILD(named, thread)},
    {NAME("config"), CHILD(named, config)}, {NAME("opt"), CHILD(named, opt)},
    {NAME("tcache"), CHILD(named, tcache)},
    {NAME("arena"), CHILD(indexed, arena)},
    {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)},
    {NAME("stats"), CHILD(named, stats)},
    {NAME("approximate_stats"), CHILD(named, approximate_stats)},
    {NAME("experimental"), CHILD(named, experimental)}};
static const ctl_named_node_t super_root_node[] = {
    {NAME(""), CHILD(named, root)}};

#undef NAME
#undef CHILD
#undef CTL
#undef CTL_EXTERNAL
#undef INDEX

/******************************************************************************/

static void
ctl_refresh(tsdn_t *tsdn) {
	malloc_mutex_assert_owner(tsdn, &ctl_mtx);
	ctl_arena_t *ctl_sarena = ctl_arenas_refresh(tsdn);
	ctl_stats_refresh(tsdn, ctl_sarena);
	ctl_arenas_epoch_advance();
}

static bool
ctl_init(tsd_t *tsd) {
	bool    ret;
	tsdn_t *tsdn = tsd_tsdn(tsd);

	malloc_mutex_lock(tsdn, &ctl_mtx);
	if (!ctl_initialized) {
		if (ctl_stats_init(tsdn)) {
			ret = true;
			goto label_return;
		}

		if (ctl_arenas_init(tsd)) {
			ret = true;
			goto label_return;
		}
		ctl_refresh(tsdn);

		ctl_initialized = true;
	}

	ret = false;
label_return:
	malloc_mutex_unlock(tsdn, &ctl_mtx);
	return ret;
}

static int
ctl_lookup(tsdn_t *tsdn, const ctl_named_node_t *starting_node,
    const char *name, const ctl_named_node_t **ending_nodep, size_t *mibp,
    size_t *depthp) {
	int                     ret;
	const char             *elm, *tdot, *dot;
	size_t                  elen, i, j;
	const ctl_named_node_t *node;

	elm = name;
	/* Equivalent to strchrnul(). */
	dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0');
	elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
	if (elen == 0) {
		ret = ENOENT;
		goto label_return;
	}
	node = starting_node;
	for (i = 0; i < *depthp; i++) {
		assert(node);
		assert(node->nchildren > 0);
		if (ctl_named_node(node->children) != NULL) {
			const ctl_named_node_t *pnode = node;

			/* Children are named. */
			for (j = 0; j < node->nchildren; j++) {
				const ctl_named_node_t *child =
				    ctl_named_children(node, j);
				if (strlen(child->name) == elen
				    && strncmp(elm, child->name, elen) == 0) {
					node = child;
					mibp[i] = j;
					break;
				}
			}
			if (node == pnode) {
				ret = ENOENT;
				goto label_return;
			}
		} else {
			uintmax_t                 index;
			const ctl_indexed_node_t *inode;

			/* Children are indexed. */
			index = malloc_strtoumax(elm, NULL, 10);
			if (index == UINTMAX_MAX || index > SIZE_T_MAX) {
				ret = ENOENT;
				goto label_return;
			}

			inode = ctl_indexed_node(node->children);
			node = inode->index(tsdn, mibp, *depthp, (size_t)index);
			if (node == NULL) {
				ret = ENOENT;
				goto label_return;
			}

			mibp[i] = (size_t)index;
		}

		/* Reached the end? */
		if (node->ctl != NULL || *dot == '\0') {
			/* Terminal node. */
			if (*dot != '\0') {
				/*
				 * The name contains more elements than are
				 * in this path through the tree.
				 */
				ret = ENOENT;
				goto label_return;
			}
			/* Complete lookup successful. */
			*depthp = i + 1;
			break;
		}

		/* Update elm. */
		elm = &dot[1];
		dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot
		                                          : strchr(elm, '\0');
		elen = (size_t)((uintptr_t)dot - (uintptr_t)elm);
	}
	if (ending_nodep != NULL) {
		*ending_nodep = node;
	}

	ret = 0;
label_return:
	return ret;
}

int
ctl_byname(tsd_t *tsd, const char *name, void *oldp, size_t *oldlenp,
    void *newp, size_t newlen) {
	int                     ret;
	size_t                  depth;
	size_t                  mib[CTL_MAX_DEPTH];
	const ctl_named_node_t *node;

	if (!ctl_initialized && ctl_init(tsd)) {
		ret = EAGAIN;
		goto label_return;
	}

	depth = CTL_MAX_DEPTH;
	ret = ctl_lookup(
	    tsd_tsdn(tsd), super_root_node, name, &node, mib, &depth);
	if (ret != 0) {
		goto label_return;
	}

	if (node != NULL && node->ctl) {
		ret = node->ctl(tsd, mib, depth, oldp, oldlenp, newp, newlen);
	} else {
		/* The name refers to a partial path through the ctl tree. */
		ret = ENOENT;
	}

label_return:
	return (ret);
}

static const ctl_named_node_t *
arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
	return ctl_arena_i_indexable(tsdn, i) ? super_arena_i_node : NULL;
}

static const ctl_named_node_t *
arenas_bin_i_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
	if (i >= SC_NBINS) {
		return NULL;
	}
	return super_arenas_bin_i_node;
}

static const ctl_named_node_t *
arenas_lextent_i_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
	if (i >= SC_NSIZES - SC_NBINS) {
		return NULL;
	}
	return super_arenas_lextent_i_node;
}

int
ctl_nametomib(tsd_t *tsd, const char *name, size_t *mibp, size_t *miblenp) {
	int ret;

	if (!ctl_initialized && ctl_init(tsd)) {
		ret = EAGAIN;
		goto label_return;
	}

	ret = ctl_lookup(
	    tsd_tsdn(tsd), super_root_node, name, NULL, mibp, miblenp);
label_return:
	return (ret);
}

static int
ctl_lookupbymib(tsdn_t *tsdn, const ctl_named_node_t **ending_nodep,
    const size_t *mib, size_t miblen) {
	int ret;

	const ctl_named_node_t *node = super_root_node;
	for (size_t i = 0; i < miblen; i++) {
		assert(node);
		assert(node->nchildren > 0);
		if (ctl_named_node(node->children) != NULL) {
			/* Children are named. */
			if (node->nchildren <= mib[i]) {
				ret = ENOENT;
				goto label_return;
			}
			node = ctl_named_children(node, mib[i]);
		} else {
			const ctl_indexed_node_t *inode;

			/* Indexed element. */
			inode = ctl_indexed_node(node->children);
			node = inode->index(tsdn, mib, miblen, mib[i]);
			if (node == NULL) {
				ret = ENOENT;
				goto label_return;
			}
		}
	}
	assert(ending_nodep != NULL);
	*ending_nodep = node;
	ret = 0;

label_return:
	return (ret);
}

int
ctl_bymib(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
    size_t *oldlenp, void *newp, size_t newlen) {
	int                     ret;
	const ctl_named_node_t *node;

	if (!ctl_initialized && ctl_init(tsd)) {
		ret = EAGAIN;
		goto label_return;
	}

	ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
	if (ret != 0) {
		goto label_return;
	}

	/* Call the ctl function. */
	if (node && node->ctl) {
		ret = node->ctl(tsd, mib, miblen, oldp, oldlenp, newp, newlen);
	} else {
		/* Partial MIB. */
		ret = ENOENT;
	}

label_return:
	return (ret);
}

int
ctl_mibnametomib(
    tsd_t *tsd, size_t *mib, size_t miblen, const char *name, size_t *miblenp) {
	int                     ret;
	const ctl_named_node_t *node;

	if (!ctl_initialized && ctl_init(tsd)) {
		ret = EAGAIN;
		goto label_return;
	}

	ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
	if (ret != 0) {
		goto label_return;
	}
	if (node == NULL || node->ctl != NULL) {
		ret = ENOENT;
		goto label_return;
	}

	assert(miblenp != NULL);
	assert(*miblenp >= miblen);
	*miblenp -= miblen;
	ret = ctl_lookup(
	    tsd_tsdn(tsd), node, name, NULL, mib + miblen, miblenp);
	*miblenp += miblen;
label_return:
	return (ret);
}

int
ctl_bymibname(tsd_t *tsd, size_t *mib, size_t miblen, const char *name,
    size_t *miblenp, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
	int                     ret;
	const ctl_named_node_t *node;

	if (!ctl_initialized && ctl_init(tsd)) {
		ret = EAGAIN;
		goto label_return;
	}

	ret = ctl_lookupbymib(tsd_tsdn(tsd), &node, mib, miblen);
	if (ret != 0) {
		goto label_return;
	}
	if (node == NULL || node->ctl != NULL) {
		ret = ENOENT;
		goto label_return;
	}

	assert(miblenp != NULL);
	assert(*miblenp >= miblen);
	*miblenp -= miblen;
	/*
	 * The same node supplies the starting node and stores the ending node.
	 */
	ret = ctl_lookup(
	    tsd_tsdn(tsd), node, name, &node, mib + miblen, miblenp);
	*miblenp += miblen;
	if (ret != 0) {
		goto label_return;
	}

	if (node != NULL && node->ctl) {
		ret = node->ctl(
		    tsd, mib, *miblenp, oldp, oldlenp, newp, newlen);
	} else {
		/* The name refers to a partial path through the ctl tree. */
		ret = ENOENT;
	}

label_return:
	return (ret);
}

bool
ctl_boot(void) {
	if (malloc_mutex_init(&ctl_mtx, "ctl", WITNESS_RANK_CTL,
	        malloc_mutex_rank_exclusive)) {
		return true;
	}

	ctl_initialized = false;

	return false;
}

void
ctl_prefork(tsdn_t *tsdn) {
	malloc_mutex_prefork(tsdn, &ctl_mtx);
}

void
ctl_postfork_parent(tsdn_t *tsdn) {
	malloc_mutex_postfork_parent(tsdn, &ctl_mtx);
}

void
ctl_postfork_child(tsdn_t *tsdn) {
	malloc_mutex_postfork_child(tsdn, &ctl_mtx);
}

void
ctl_mtx_lock(tsdn_t *tsdn) {
	malloc_mutex_lock(tsdn, &ctl_mtx);
}

void
ctl_mtx_unlock(tsdn_t *tsdn) {
	malloc_mutex_unlock(tsdn, &ctl_mtx);
}

void
ctl_mtx_assert_held(tsdn_t *tsdn) {
	malloc_mutex_assert_owner(tsdn, &ctl_mtx);
}

void
ctl_mtx_prof_read(tsdn_t *tsdn, mutex_prof_data_t *mutex_prof_data) {
	malloc_mutex_prof_read(tsdn, mutex_prof_data, &ctl_mtx);
}

void
ctl_mtx_prof_data_reset(tsdn_t *tsdn) {
	malloc_mutex_lock(tsdn, &ctl_mtx);
	malloc_mutex_prof_data_reset(tsdn, &ctl_mtx);
	malloc_mutex_unlock(tsdn, &ctl_mtx);
}

/******************************************************************************/

static int
version_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
    size_t *oldlenp, void *newp, size_t newlen) {
	int         ret;
	const char *oldval;

	READONLY();
	oldval = JEMALLOC_VERSION;
	READ(oldval, const char *);

	ret = 0;
label_return:
	return ret;
}

static int
epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp,
    size_t *oldlenp, void *newp, size_t newlen) {
	int             ret;
	UNUSED uint64_t newval;

	malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
	WRITE(newval, uint64_t);
	if (newp != NULL) {
		ctl_refresh(tsd_tsdn(tsd));
	}
	uint64_t epoch = ctl_arenas_epoch_get();
	READ(epoch, uint64_t);

	ret = 0;
label_return:
	malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx);
	return ret;
}

/******************************************************************************/

static const ctl_named_node_t *
stats_arenas_i_bins_j_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
	if (j >= SC_NBINS) {
		return NULL;
	}
	return super_stats_arenas_i_bins_j_node;
}


static const ctl_named_node_t *
stats_arenas_i_lextents_j_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
	if (j >= SC_NSIZES - SC_NBINS) {
		return NULL;
	}
	return super_stats_arenas_i_lextents_j_node;
}


static const ctl_named_node_t *
stats_arenas_i_extents_j_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
	if (j >= SC_NPSIZES) {
		return NULL;
	}
	return super_stats_arenas_i_extents_j_node;
}


static const ctl_named_node_t *
stats_arenas_i_hpa_shard_nonfull_slabs_j_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
	if (j >= PSSET_NPSIZES) {
		return NULL;
	}
	return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node;
}


static const ctl_named_node_t *
stats_arenas_i_hpa_shard_alloc_j_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
	if (j > SEC_MAX_NALLOCS) {
		return NULL;
	}
	return super_stats_arenas_i_hpa_shard_alloc_j_node;
}


static const ctl_named_node_t *
stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
	const ctl_named_node_t *ret;

	malloc_mutex_lock(tsdn, &ctl_mtx);
	if (ctl_arenas_i_verify(i)) {
		ret = NULL;
		goto label_return;
	}

	ret = super_stats_arenas_i_node;
label_return:
	malloc_mutex_unlock(tsdn, &ctl_mtx);
	return ret;
}


static const ctl_named_node_t *
prof_stats_bins_i_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
	if (!(config_prof && opt_prof && opt_prof_stats)) {
		return NULL;
	}
	if (i >= SC_NBINS) {
		return NULL;
	}
	return super_prof_stats_bins_i_node;
}

static const ctl_named_node_t *
prof_stats_lextents_i_index(
    tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) {
	if (!(config_prof && opt_prof && opt_prof_stats)) {
		return NULL;
	}
	if (i >= SC_NSIZES - SC_NBINS) {
		return NULL;
	}
	return super_prof_stats_lextents_i_node;
}