From 9fd9c876bb99acc957f8ec411837138a9b588a1e Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Sun, 6 Dec 2020 12:49:03 -0800 Subject: [PATCH] psset: keep aggregate stats. This will let us quickly query these stats to make purging decisions quickly. --- include/jemalloc/internal/psset.h | 24 ++++++++++++-- src/psset.c | 52 ++++++++++++++++++++++++------- test/unit/psset.c | 1 + 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 6e08e8ba..d2a8b24a 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -8,9 +8,6 @@ * a collection of page-slabs (the intent being that they are backed by * hugepages, or at least could be), and handles allocation and deallocation * requests. - * - * It has the same synchronization guarantees as the eset; stats queries don't - * need any external synchronization, everything else does. */ /* @@ -60,6 +57,12 @@ struct psset_s { */ hpdata_age_heap_t pageslabs[PSSET_NPSIZES]; bitmap_t bitmap[BITMAP_GROUPS(PSSET_NPSIZES)]; + /* + * The sum of all bin stats in stats. This lets us quickly answer + * queries for the number of dirty, active, and retained pages in the + * entire set. + */ + psset_bin_stats_t merged_stats; psset_stats_t stats; /* * Slabs with no active allocations, but which are allowed to serve new @@ -92,4 +95,19 @@ hpdata_t *psset_pick_hugify(psset_t *psset); void psset_insert(psset_t *psset, hpdata_t *ps); void psset_remove(psset_t *psset, hpdata_t *ps); +static inline size_t +psset_npageslabs(psset_t *psset) { + return psset->merged_stats.npageslabs; +} + +static inline size_t +psset_nactive(psset_t *psset) { + return psset->merged_stats.nactive; +} + +static inline size_t +psset_ndirty(psset_t *psset) { + return psset->merged_stats.ndirty; +} + #endif /* JEMALLOC_INTERNAL_PSSET_H */ diff --git a/src/psset.c b/src/psset.c index bb51e21e..66d37397 100644 --- a/src/psset.c +++ b/src/psset.c @@ -14,6 +14,7 @@ psset_init(psset_t *psset) { hpdata_age_heap_new(&psset->pageslabs[i]); } bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true); + memset(&psset->merged_stats, 0, sizeof(psset->merged_stats)); memset(&psset->stats, 0, sizeof(psset->stats)); hpdata_empty_list_init(&psset->empty); hpdata_purge_list_init(&psset->to_purge); @@ -52,23 +53,48 @@ psset_stats_accum(psset_stats_t *dst, psset_stats_t *src) { * ensure we don't miss any heap modification operations. */ JEMALLOC_ALWAYS_INLINE void -psset_bin_stats_insert_remove(psset_bin_stats_t *binstats, hpdata_t *ps, - bool insert) { +psset_bin_stats_insert_remove(psset_t *psset, psset_bin_stats_t *binstats, + hpdata_t *ps, bool insert) { size_t mul = insert ? (size_t)1 : (size_t)-1; size_t huge_idx = (size_t)hpdata_huge_get(ps); + binstats[huge_idx].npageslabs += mul * 1; binstats[huge_idx].nactive += mul * hpdata_nactive_get(ps); binstats[huge_idx].ndirty += mul * hpdata_ndirty_get(ps); + + psset->merged_stats.npageslabs += mul * 1; + psset->merged_stats.nactive += mul * hpdata_nactive_get(ps); + psset->merged_stats.ndirty += mul * hpdata_ndirty_get(ps); + + if (config_debug) { + psset_bin_stats_t check_stats = {0}; + for (size_t huge = 0; huge <= 1; huge++) { + psset_bin_stats_accum(&check_stats, + &psset->stats.full_slabs[huge]); + psset_bin_stats_accum(&check_stats, + &psset->stats.empty_slabs[huge]); + for (pszind_t pind = 0; pind < PSSET_NPSIZES; pind++) { + psset_bin_stats_accum(&check_stats, + &psset->stats.nonfull_slabs[pind][huge]); + } + } + assert(psset->merged_stats.npageslabs + == check_stats.npageslabs); + assert(psset->merged_stats.nactive == check_stats.nactive); + assert(psset->merged_stats.ndirty == check_stats.ndirty); + } } static void -psset_bin_stats_insert(psset_bin_stats_t *binstats, hpdata_t *ps) { - psset_bin_stats_insert_remove(binstats, ps, true); +psset_bin_stats_insert(psset_t *psset, psset_bin_stats_t *binstats, + hpdata_t *ps) { + psset_bin_stats_insert_remove(psset, binstats, ps, true); } static void -psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) { - psset_bin_stats_insert_remove(binstats, ps, false); +psset_bin_stats_remove(psset_t *psset, psset_bin_stats_t *binstats, + hpdata_t *ps) { + psset_bin_stats_insert_remove(psset, binstats, ps, false); } static void @@ -90,9 +116,9 @@ psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) { static void psset_stats_insert(psset_t* psset, hpdata_t *ps) { if (hpdata_empty(ps)) { - psset_bin_stats_insert(psset->stats.empty_slabs, ps); + psset_bin_stats_insert(psset, psset->stats.empty_slabs, ps); } else if (hpdata_full(ps)) { - psset_bin_stats_insert(psset->stats.full_slabs, ps); + psset_bin_stats_insert(psset, psset->stats.full_slabs, ps); } else { size_t longest_free_range = hpdata_longest_free_range_get(ps); @@ -100,16 +126,17 @@ psset_stats_insert(psset_t* psset, hpdata_t *ps) { longest_free_range << LG_PAGE)); assert(pind < PSSET_NPSIZES); - psset_bin_stats_insert(psset->stats.nonfull_slabs[pind], ps); + psset_bin_stats_insert(psset, psset->stats.nonfull_slabs[pind], + ps); } } static void psset_stats_remove(psset_t *psset, hpdata_t *ps) { if (hpdata_empty(ps)) { - psset_bin_stats_remove(psset->stats.empty_slabs, ps); + psset_bin_stats_remove(psset, psset->stats.empty_slabs, ps); } else if (hpdata_full(ps)) { - psset_bin_stats_remove(psset->stats.full_slabs, ps); + psset_bin_stats_remove(psset, psset->stats.full_slabs, ps); } else { size_t longest_free_range = hpdata_longest_free_range_get(ps); @@ -117,7 +144,8 @@ psset_stats_remove(psset_t *psset, hpdata_t *ps) { longest_free_range << LG_PAGE)); assert(pind < PSSET_NPSIZES); - psset_bin_stats_remove(psset->stats.nonfull_slabs[pind], ps); + psset_bin_stats_remove(psset, psset->stats.nonfull_slabs[pind], + ps); } } diff --git a/test/unit/psset.c b/test/unit/psset.c index f5e1bad5..b93dfbfe 100644 --- a/test/unit/psset.c +++ b/test/unit/psset.c @@ -374,6 +374,7 @@ stats_expect(psset_t *psset, size_t nactive) { stats_expect_empty(&psset->stats.nonfull_slabs[i][0]); } } + expect_zu_eq(nactive, psset_nactive(psset), ""); } TEST_BEGIN(test_stats) {