diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 5b33769f..eac4a631 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -150,9 +150,10 @@ arena_decay_extent(tsdn_t *tsdn,arena_t *arena, ehooks_t *ehooks, /* Update stats accordingly. */ LOCKEDINT_MTX_LOCK(tsdn, arena->stats.mtx); locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->decay_dirty.stats->nmadvise, 1); + &arena->pa_shard.stats->decay_dirty.nmadvise, 1); locked_inc_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE); + &arena->pa_shard.stats->decay_dirty.purged, + extent_size >> LG_PAGE); locked_dec_zu(tsdn, LOCKEDINT_MTX(arena->stats.mtx), &arena->stats.mapped, extent_size); LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx); diff --git a/include/jemalloc/internal/arena_stats.h b/include/jemalloc/internal/arena_stats.h index 0a1ec734..82996b8b 100644 --- a/include/jemalloc/internal/arena_stats.h +++ b/include/jemalloc/internal/arena_stats.h @@ -37,16 +37,6 @@ struct arena_stats_large_s { size_t curlextents; /* Derived. */ }; -typedef struct arena_stats_decay_s arena_stats_decay_t; -struct arena_stats_decay_s { - /* Total number of purge sweeps. */ - locked_u64_t npurge; - /* Total number of madvise calls made. */ - locked_u64_t nmadvise; - /* Total number of pages purged. */ - locked_u64_t purged; -}; - typedef struct arena_stats_extents_s arena_stats_extents_t; struct arena_stats_extents_s { /* @@ -87,9 +77,6 @@ struct arena_stats_s { /* Number of edata_t structs allocated by base, but not being used. */ atomic_zu_t edata_avail; - arena_stats_decay_t decay_dirty; - arena_stats_decay_t decay_muzzy; - atomic_zu_t base; /* Derived. */ atomic_zu_t internal; atomic_zu_t resident; /* Derived. */ diff --git a/include/jemalloc/internal/arena_structs.h b/include/jemalloc/internal/arena_structs.h index ed16337b..7d312343 100644 --- a/include/jemalloc/internal/arena_structs.h +++ b/include/jemalloc/internal/arena_structs.h @@ -73,7 +73,7 @@ struct arena_decay_s { * arena and ctl code. * * Synchronization: Same as associated arena's stats field. */ - arena_stats_decay_t *stats; + pa_shard_decay_stats_t *stats; /* Peak number of pages in associated extents. Used for debug only. */ uint64_t ceil_npages; }; diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 3b1a7659..1b9e58c4 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -3,14 +3,27 @@ #include "jemalloc/internal/ecache.h" #include "jemalloc/internal/edata_cache.h" +#include "jemalloc/internal/lockedint.h" /* * The page allocator; responsible for acquiring pages of memory for * allocations. */ +typedef struct pa_shard_decay_stats_s pa_shard_decay_stats_t; +struct pa_shard_decay_stats_s { + /* Total number of purge sweeps. */ + locked_u64_t npurge; + /* Total number of madvise calls made. */ + locked_u64_t nmadvise; + /* Total number of pages purged. */ + locked_u64_t purged; +}; + typedef struct pa_shard_stats_s pa_shard_stats_t; struct pa_shard_stats_s { + pa_shard_decay_stats_t decay_dirty; + pa_shard_decay_stats_t decay_muzzy; /* VM space had to be leaked (undocumented). Normally 0. */ atomic_zu_t abandoned_vm; }; diff --git a/src/arena.c b/src/arena.c index d4e200cf..a29dc937 100644 --- a/src/arena.c +++ b/src/arena.c @@ -105,25 +105,33 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, atomic_load_zu(&arena->pa_shard.edata_cache.count, ATOMIC_RELAXED), ATOMIC_RELAXED); - locked_inc_u64_unsynchronized(&astats->decay_dirty.npurge, + /* Dirty pa_shard_decay_stats_t */ + locked_inc_u64_unsynchronized( + &astats->pa_shard_stats.decay_dirty.npurge, locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.decay_dirty.npurge)); - locked_inc_u64_unsynchronized(&astats->decay_dirty.nmadvise, + &arena->pa_shard.stats->decay_dirty.npurge)); + locked_inc_u64_unsynchronized( + &astats->pa_shard_stats.decay_dirty.nmadvise, locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.decay_dirty.nmadvise)); - locked_inc_u64_unsynchronized(&astats->decay_dirty.purged, + &arena->pa_shard.stats->decay_dirty.nmadvise)); + locked_inc_u64_unsynchronized( + &astats->pa_shard_stats.decay_dirty.purged, locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.decay_dirty.purged)); + &arena->pa_shard.stats->decay_dirty.purged)); - locked_inc_u64_unsynchronized(&astats->decay_muzzy.npurge, + /* Muzzy pa_shard_decay_stats_t */ + locked_inc_u64_unsynchronized( + &astats->pa_shard_stats.decay_muzzy.npurge, locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.decay_muzzy.npurge)); - locked_inc_u64_unsynchronized(&astats->decay_muzzy.nmadvise, + &arena->pa_shard.stats->decay_muzzy.npurge)); + locked_inc_u64_unsynchronized( + &astats->pa_shard_stats.decay_muzzy.nmadvise, locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.decay_muzzy.nmadvise)); - locked_inc_u64_unsynchronized(&astats->decay_muzzy.purged, + &arena->pa_shard.stats->decay_muzzy.nmadvise)); + locked_inc_u64_unsynchronized( + &astats->pa_shard_stats.decay_muzzy.purged, locked_read_u64(tsdn, LOCKEDINT_MTX(arena->stats.mtx), - &arena->stats.decay_muzzy.purged)); + &arena->pa_shard.stats->decay_muzzy.purged)); atomic_load_add_store_zu(&astats->base, base_allocated); atomic_load_add_store_zu(&astats->internal, arena_internal_get(arena)); @@ -695,7 +703,7 @@ arena_decay_reinit(arena_decay_t *decay, ssize_t decay_ms) { static bool arena_decay_init(arena_decay_t *decay, ssize_t decay_ms, - arena_stats_decay_t *stats) { + pa_shard_decay_stats_t *stats) { if (config_debug) { for (size_t i = 0; i < sizeof(arena_decay_t); i++) { assert(((char *)decay)[i] == 0); @@ -708,7 +716,6 @@ arena_decay_init(arena_decay_t *decay, ssize_t decay_ms, } decay->purging = false; arena_decay_reinit(decay, decay_ms); - /* Memory is zeroed, so there is no need to clear stats. */ if (config_stats) { decay->stats = stats; } @@ -2044,11 +2051,13 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } if (arena_decay_init(&arena->decay_dirty, - arena_dirty_decay_ms_default_get(), &arena->stats.decay_dirty)) { + arena_dirty_decay_ms_default_get(), + &arena->pa_shard.stats->decay_dirty)) { goto label_error; } if (arena_decay_init(&arena->decay_muzzy, - arena_muzzy_decay_ms_default_get(), &arena->stats.decay_muzzy)) { + arena_muzzy_decay_ms_default_get(), + &arena->pa_shard.stats->decay_muzzy)) { goto label_error; } diff --git a/src/ctl.c b/src/ctl.c index 56d30000..26d86da0 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -869,19 +869,25 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, &astats->astats.edata_avail); } - ctl_accum_locked_u64(&sdstats->astats.decay_dirty.npurge, - &astats->astats.decay_dirty.npurge); - ctl_accum_locked_u64(&sdstats->astats.decay_dirty.nmadvise, - &astats->astats.decay_dirty.nmadvise); - ctl_accum_locked_u64(&sdstats->astats.decay_dirty.purged, - &astats->astats.decay_dirty.purged); + ctl_accum_locked_u64( + &sdstats->astats.pa_shard_stats.decay_dirty.npurge, + &astats->astats.pa_shard_stats.decay_dirty.npurge); + ctl_accum_locked_u64( + &sdstats->astats.pa_shard_stats.decay_dirty.nmadvise, + &astats->astats.pa_shard_stats.decay_dirty.nmadvise); + ctl_accum_locked_u64( + &sdstats->astats.pa_shard_stats.decay_dirty.purged, + &astats->astats.pa_shard_stats.decay_dirty.purged); - ctl_accum_locked_u64(&sdstats->astats.decay_muzzy.npurge, - &astats->astats.decay_muzzy.npurge); - ctl_accum_locked_u64(&sdstats->astats.decay_muzzy.nmadvise, - &astats->astats.decay_muzzy.nmadvise); - ctl_accum_locked_u64(&sdstats->astats.decay_muzzy.purged, - &astats->astats.decay_muzzy.purged); + ctl_accum_locked_u64( + &sdstats->astats.pa_shard_stats.decay_muzzy.npurge, + &astats->astats.pa_shard_stats.decay_muzzy.npurge); + ctl_accum_locked_u64( + &sdstats->astats.pa_shard_stats.decay_muzzy.nmadvise, + &astats->astats.pa_shard_stats.decay_muzzy.nmadvise); + ctl_accum_locked_u64( + &sdstats->astats.pa_shard_stats.decay_muzzy.purged, + &astats->astats.pa_shard_stats.decay_muzzy.purged); #define OP(mtx) malloc_mutex_prof_merge( \ &(sdstats->astats.mutex_prof_data[ \ @@ -2920,23 +2926,29 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.decay_dirty.npurge), uint64_t) + &arenas_i(mib[2])->astats->astats.pa_shard_stats.decay_dirty.npurge), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t) + &arenas_i(mib[2])->astats->astats.pa_shard_stats.decay_dirty.nmadvise), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.decay_dirty.purged), uint64_t) + &arenas_i(mib[2])->astats->astats.pa_shard_stats.decay_dirty.purged), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), uint64_t) + &arenas_i(mib[2])->astats->astats.pa_shard_stats.decay_muzzy.npurge), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t) + &arenas_i(mib[2])->astats->astats.pa_shard_stats.decay_muzzy.nmadvise), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, locked_read_u64_unsynchronized( - &arenas_i(mib[2])->astats->astats.decay_muzzy.purged), uint64_t) + &arenas_i(mib[2])->astats->astats.pa_shard_stats.decay_muzzy.purged), + uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_base, atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED),