From 9c1a484e1de990678986b5e4a6c7768dba25e0b2 Mon Sep 17 00:00:00 2001 From: Bin Liu Date: Tue, 19 May 2026 00:11:15 -0700 Subject: [PATCH] Use SEC in PAC to reduce lock contention on the ecaches Add a small extent cache in front of the PAC ecaches. Allocs and dallocs that fit are served from per-shard SEC bins without taking the ecache mutex; overflow falls through to the backing ecaches, including ecache_pinned for pinned extents. The feature is gated behind experimental_pac_sec_nshards (default 0, disabled). To support independent HPA and PAC SEC instances, sec_alloc/sec_dalloc/sec_fill take an explicit shard argument, with HPA and PAC using separate TSD shard slots. --- Makefile.in | 1 + .../internal/jemalloc_internal_externs.h | 1 + include/jemalloc/internal/mutex_prof.h | 3 +- include/jemalloc/internal/pac.h | 15 + include/jemalloc/internal/sec.h | 37 +- include/jemalloc/internal/tsd_internals.h | 2 + src/conf.c | 11 + src/ctl.c | 51 +++ src/hpa.c | 38 +- src/jemalloc.c | 3 + src/pa.c | 1 + src/pa_extra.c | 52 ++- src/pac.c | 96 +++++ src/sec.c | 159 +++++-- src/stats.c | 38 ++ test/unit/json_stats.c | 2 +- test/unit/mallctl.c | 3 + test/unit/pac_sec_integration.c | 387 ++++++++++++++++++ test/unit/sec.c | 188 ++++++--- 19 files changed, 986 insertions(+), 102 deletions(-) create mode 100644 test/unit/pac_sec_integration.c diff --git a/Makefile.in b/Makefile.in index 16d6a820..a70bdf53 100644 --- a/Makefile.in +++ b/Makefile.in @@ -272,6 +272,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/oversize_threshold.c \ $(srcroot)test/unit/pa.c \ $(srcroot)test/unit/pac.c \ + $(srcroot)test/unit/pac_sec_integration.c \ $(srcroot)test/unit/pack.c \ $(srcroot)test/unit/pages.c \ $(srcroot)test/unit/peak.c \ diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index 4d2d18fe..c5af5cc0 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -21,6 +21,7 @@ extern bool opt_confirm_conf; extern bool opt_hpa; extern hpa_shard_opts_t opt_hpa_opts; extern sec_opts_t opt_hpa_sec_opts; +extern sec_opts_t opt_pac_sec_opts; extern const char *opt_junk; extern bool opt_junk_alloc; diff --git a/include/jemalloc/internal/mutex_prof.h b/include/jemalloc/internal/mutex_prof.h index 37f6a377..590b46c3 100644 --- a/include/jemalloc/internal/mutex_prof.h +++ b/include/jemalloc/internal/mutex_prof.h @@ -37,7 +37,8 @@ typedef enum { OP(tcache_list) \ OP(hpa_shard) \ OP(hpa_shard_grow) \ - OP(hpa_sec) + OP(hpa_sec) \ + OP(pac_sec) typedef enum { #define OP(mtx) arena_prof_mutex_##mtx, diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h index d58a6144..b43e5dd2 100644 --- a/include/jemalloc/internal/pac.h +++ b/include/jemalloc/internal/pac.h @@ -8,6 +8,7 @@ #include "jemalloc/internal/edata_cache.h" #include "jemalloc/internal/exp_grow.h" #include "jemalloc/internal/lockedint.h" +#include "jemalloc/internal/sec.h" #include "jemalloc/internal/tsd_types.h" #include "san_bump.h" @@ -84,12 +85,24 @@ struct pac_stats_s { /* VM space had to be leaked (undocumented). Normally 0. */ atomic_zu_t abandoned_vm; + + /* PAC SEC stats. Derived. */ + sec_stats_t pac_sec_stats; }; typedef struct pac_s pac_t; struct pac_s { + /* Small extent cache in front of PAC ecaches to reduce contention. */ + sec_t sec; + /* + * Runtime gate for PAC SEC. 0 disables (when SEC is not configured or + * dirty_decay_ms == 0); otherwise mirrors sec.opts.max_alloc. + */ + atomic_zu_t sec_max_alloc; + /* True once pinned memory has been seen. */ atomic_b_t has_pinned; + /* * Collections of extents that were previously allocated. These are * used when allocating extents, in an attempt to re-use address space. @@ -236,4 +249,6 @@ ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state); void pac_destroy(tsdn_t *tsdn, pac_t *pac); +void pac_sec_flush(tsdn_t *tsdn, pac_t *pac); + #endif /* JEMALLOC_INTERNAL_PAC_H */ diff --git a/include/jemalloc/internal/sec.h b/include/jemalloc/internal/sec.h index 6dd2a293..f2abe99d 100644 --- a/include/jemalloc/internal/sec.h +++ b/include/jemalloc/internal/sec.h @@ -35,11 +35,21 @@ typedef struct sec_stats_s sec_stats_t; struct sec_stats_s { /* Sum of bytes_cur across all shards. */ size_t bytes; + /* Subset of bytes that are pinned. */ + size_t bytes_pinned; /* Totals of bin_stats. */ sec_bin_stats_t total; }; +typedef struct sec_pszind_stats_s sec_pszind_stats_t; +struct sec_pszind_stats_s { + size_t nextents; + size_t bytes; + size_t nextents_pinned; + size_t bytes_pinned; +}; + static inline void sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) { dst->nmisses += src->nmisses; @@ -52,6 +62,7 @@ sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) { static inline void sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) { dst->bytes += src->bytes; + dst->bytes_pinned += src->bytes_pinned; sec_bin_stats_accum(&dst->total, &src->total); } @@ -67,6 +78,7 @@ struct sec_bin_s { * Number of bytes in this particular bin. */ atomic_zu_t bytes_cur; + atomic_zu_t bytes_pinned_cur; edata_list_active_t freelist; atomic_zu_t nmisses; atomic_zu_t nhits; @@ -88,8 +100,8 @@ sec_is_used(const sec_t *sec) { } static inline bool -sec_size_supported(sec_t *sec, size_t size) { - return sec_is_used(sec) && size <= sec->opts.max_alloc; +sec_size_supported(const sec_t *sec, size_t size) { + return size <= sec->opts.max_alloc; } /* Min number of extents we will allocate when expanding the SEC. */ @@ -111,12 +123,21 @@ sec_size_supported(sec_t *sec, size_t size) { void sec_calc_nallocs_for_size( sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs); -/* If sec does not have extent available, it will return NULL. */ -edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size); +/* + * Lazily picks (and caches in *idxp) a shard for the calling thread. Different + * SEC instances pass independent per-thread uint8_t slots, initialized to + * (uint8_t)-1. + */ +uint8_t sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp); + +/* Callers must ensure sec_size_supported(sec, size). */ +edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard); void sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, - edata_list_active_t *result, size_t nallocs); + edata_list_active_t *result, size_t nallocs, uint8_t shard); /* + * Callers must ensure sec_size_supported(sec, edata_size). + * * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty * if there are extents that need to be flushed from cache. Please note, that * if we need to flush, extent(s) returned in the list to be deallocated @@ -124,7 +145,8 @@ void sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, * considered "hot" and preserved in the cache, while "colder" ones are * returned). */ -void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list); +void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list, + uint8_t shard); bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts); @@ -138,6 +160,9 @@ void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush); * split), which simplifies the stats management. */ void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats); +void sec_stats_merge_pszind( + tsdn_t *tsdn, const sec_t *sec, pszind_t pszind, + sec_pszind_stats_t *stats); void sec_mutex_stats_read( tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data); diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index e25c0b85..b318c464 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -83,6 +83,7 @@ typedef void (*test_callback_t)(int *); O(arena, arena_t *, arena_t *) \ O(arena_decay_ticker, ticker_geom_t, ticker_geom_t) \ O(sec_shard, uint8_t, uint8_t) \ + O(pac_sec_shard, uint8_t, uint8_t) \ O(binshards, tsd_binshards_t, tsd_binshards_t) \ O(peak, peak_t, peak_t) \ O(tcache_slow, tcache_slow_t, tcache_slow_t) \ @@ -102,6 +103,7 @@ typedef void (*test_callback_t)(int *); /* arena */ NULL, /* arena_decay_ticker */ \ TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE), \ /* sec_shard */ (uint8_t) - 1, \ + /* pac_sec_shard */ (uint8_t) - 1, \ /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER, \ /* peak */ PEAK_INITIALIZER, /* tcache_slow */ \ TCACHE_SLOW_ZERO_INITIALIZER, \ diff --git a/src/conf.c b/src/conf.c index 0ca2112d..e33070b2 100644 --- a/src/conf.c +++ b/src/conf.c @@ -950,6 +950,17 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes, "hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0, CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_pac_sec_opts.nshards, + "experimental_pac_sec_nshards", 0, 0, + CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_alloc, + "experimental_pac_sec_max_alloc", PAGE, + USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN, + CONF_CHECK_MAX, true); + CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_bytes, + "experimental_pac_sec_max_bytes", + SEC_OPTS_MAX_BYTES_DEFAULT, 0, + CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true); if (CONF_MATCH("slab_sizes")) { if (CONF_MATCH_VALUE("default")) { diff --git a/src/ctl.c b/src/ctl.c index 095fc7ac..b94cf22a 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -147,6 +147,9 @@ CTL_PROTO(opt_hpa_dirty_mult) CTL_PROTO(opt_hpa_sec_nshards) CTL_PROTO(opt_hpa_sec_max_alloc) CTL_PROTO(opt_hpa_sec_max_bytes) +CTL_PROTO(opt_experimental_pac_sec_nshards) +CTL_PROTO(opt_experimental_pac_sec_max_alloc) +CTL_PROTO(opt_experimental_pac_sec_max_bytes) CTL_PROTO(opt_huge_arena_pac_thp) CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) @@ -382,6 +385,11 @@ CTL_PROTO(stats_arenas_i_hpa_sec_misses) CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_flush) CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_noflush) CTL_PROTO(stats_arenas_i_hpa_sec_overfills) +CTL_PROTO(stats_arenas_i_pac_sec_bytes) +CTL_PROTO(stats_arenas_i_pac_sec_hits) +CTL_PROTO(stats_arenas_i_pac_sec_misses) +CTL_PROTO(stats_arenas_i_pac_sec_dalloc_flush) +CTL_PROTO(stats_arenas_i_pac_sec_dalloc_noflush) INDEX_PROTO(stats_arenas_i) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) @@ -520,6 +528,12 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)}, {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)}, {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)}, {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)}, + {NAME("experimental_pac_sec_nshards"), + CTL(opt_experimental_pac_sec_nshards)}, + {NAME("experimental_pac_sec_max_alloc"), + CTL(opt_experimental_pac_sec_max_alloc)}, + {NAME("experimental_pac_sec_max_bytes"), + CTL(opt_experimental_pac_sec_max_bytes)}, {NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)}, {NAME("metadata_thp"), CTL(opt_metadata_thp)}, {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, @@ -883,6 +897,12 @@ static const ctl_named_node_t stats_arenas_i_node[] = { CTL(stats_arenas_i_hpa_sec_dalloc_noflush)}, {NAME("hpa_sec_dalloc_flush"), CTL(stats_arenas_i_hpa_sec_dalloc_flush)}, {NAME("hpa_sec_overfills"), CTL(stats_arenas_i_hpa_sec_overfills)}, + {NAME("pac_sec_bytes"), CTL(stats_arenas_i_pac_sec_bytes)}, + {NAME("pac_sec_hits"), CTL(stats_arenas_i_pac_sec_hits)}, + {NAME("pac_sec_misses"), CTL(stats_arenas_i_pac_sec_misses)}, + {NAME("pac_sec_dalloc_noflush"), + CTL(stats_arenas_i_pac_sec_dalloc_noflush)}, + {NAME("pac_sec_dalloc_flush"), CTL(stats_arenas_i_pac_sec_dalloc_flush)}, {NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, @@ -1269,6 +1289,10 @@ ctl_arena_stats_sdmerge( &sdstats->astats.pa_shard_stats.pac_stats.abandoned_vm, &astats->astats.pa_shard_stats.pac_stats.abandoned_vm); + sec_stats_accum( + &sdstats->astats.pa_shard_stats.pac_stats.pac_sec_stats, + &astats->astats.pa_shard_stats.pac_stats.pac_sec_stats); + sdstats->astats.tcache_bytes += astats->astats.tcache_bytes; sdstats->astats.tcache_stashed_bytes += astats->astats.tcache_stashed_bytes; @@ -2235,6 +2259,12 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t) CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t) CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t) CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t) +CTL_RO_NL_GEN(opt_experimental_pac_sec_nshards, + opt_pac_sec_opts.nshards, size_t) +CTL_RO_NL_GEN(opt_experimental_pac_sec_max_alloc, + opt_pac_sec_opts.max_alloc, size_t) +CTL_RO_NL_GEN(opt_experimental_pac_sec_max_bytes, + opt_pac_sec_opts.max_bytes, size_t) CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool) CTL_RO_NL_GEN( opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *) @@ -3829,6 +3859,27 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_noflush, CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_overfills, arenas_i(mib[2])->astats->hpastats.secstats.total.noverfills, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_bytes, + arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats + .pac_sec_stats.bytes, + size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_hits, + arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats + .pac_sec_stats.total.nhits, + size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_misses, + arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats + .pac_sec_stats.total.nmisses, + size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_dalloc_flush, + arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats + .pac_sec_stats.total.ndalloc_flush, + size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_dalloc_noflush, + arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats + .pac_sec_stats.total.ndalloc_noflush, + size_t) + CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, arenas_i(mib[2])->astats->allocated_small, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, diff --git a/src/hpa.c b/src/hpa.c index 5140a701..7613e4d5 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -14,6 +14,18 @@ const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"}; bool opt_experimental_hpa_start_huge_if_thp_always = true; bool opt_experimental_hpa_enforce_hugify = false; +static inline uint8_t +hpa_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { + if (sec->opts.nshards <= 1) { + return 0; + } + if (tsdn_null(tsdn)) { + return 0; + } + tsd_t *tsd = tsdn_tsd(tsdn); + return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd)); +} + JET_EXTERN bool hpa_hugepage_size_exceeds_limit(void) { return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE; @@ -945,9 +957,13 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment, && (size > shard->opts.slab_max_alloc)) { return NULL; } - edata_t *edata = sec_alloc(tsdn, &shard->sec, size); - if (edata != NULL) { - return edata; + edata_t *edata = NULL; + if (sec_size_supported(&shard->sec, size)) { + edata = sec_alloc(tsdn, &shard->sec, size, + hpa_sec_shard_pick(tsdn, &shard->sec)); + if (edata != NULL) { + return edata; + } } edata_list_active_t results; edata_list_active_init(&results); @@ -966,7 +982,8 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment, } if (nsuccess > 0) { assert(sec_size_supported(&shard->sec, size)); - sec_fill(tsdn, &shard->sec, size, &results, nsuccess); + sec_fill(tsdn, &shard->sec, size, &results, nsuccess, + hpa_sec_shard_pick(tsdn, &shard->sec)); /* Unlikely rollback in case of overfill */ if (!edata_list_active_empty(&results)) { hpa_dalloc_batch( @@ -1059,11 +1076,14 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, edata_list_active_init(&dalloc_list); edata_list_active_append(&dalloc_list, edata); - sec_dalloc(tsdn, &shard->sec, &dalloc_list); - if (edata_list_active_empty(&dalloc_list)) { - /* sec consumed the pointer */ - *deferred_work_generated = false; - return; + if (sec_size_supported(&shard->sec, edata_size_get(edata))) { + sec_dalloc(tsdn, &shard->sec, &dalloc_list, + hpa_sec_shard_pick(tsdn, &shard->sec)); + if (edata_list_active_empty(&dalloc_list)) { + /* sec consumed the pointer */ + *deferred_work_generated = false; + return; + } } /* We may have more than one pointer to flush now */ hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated); diff --git a/src/jemalloc.c b/src/jemalloc.c index 6f04da2e..7629b1af 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -198,6 +198,9 @@ size_t opt_calloc_madvise_threshold = CALLOC_MADVISE_THRESHOLD_DEFAULT; bool opt_hpa = false; hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT; sec_opts_t opt_hpa_sec_opts = SEC_OPTS_DEFAULT; +sec_opts_t opt_pac_sec_opts = {0, + (32 * 1024) > (PAGE * 2) ? (32 * 1024) : (PAGE * 2), + SEC_OPTS_MAX_BYTES_DEFAULT}; /* False should be the common case. Set to true to trigger initialization. */ bool malloc_slow = true; diff --git a/src/pa.c b/src/pa.c index a3ee51ad..a30d5781 100644 --- a/src/pa.c +++ b/src/pa.c @@ -94,6 +94,7 @@ pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) { void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) { + pac_sec_flush(tsdn, &shard->pac); if (shard->ever_used_hpa) { hpa_shard_flush(tsdn, &shard->hpa); } diff --git a/src/pa_extra.c b/src/pa_extra.c index 6e6a7aeb..dc5417d1 100644 --- a/src/pa_extra.c +++ b/src/pa_extra.c @@ -17,6 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) { void pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) { + sec_prefork2(tsdn, &shard->pac.sec); if (shard->ever_used_hpa) { hpa_shard_prefork2(tsdn, &shard->hpa); } @@ -54,6 +55,7 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) { ecache_postfork_parent(tsdn, &shard->pac.ecache_retained); ecache_postfork_parent(tsdn, &shard->pac.ecache_pinned); malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx); + sec_postfork_parent(tsdn, &shard->pac.sec); malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx); malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx); if (shard->ever_used_hpa) { @@ -69,6 +71,7 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) { ecache_postfork_child(tsdn, &shard->pac.ecache_retained); ecache_postfork_child(tsdn, &shard->pac.ecache_pinned); malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx); + sec_postfork_child(tsdn, &shard->pac.sec); malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx); malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx); if (shard->ever_used_hpa) { @@ -82,7 +85,17 @@ pa_shard_nactive(const pa_shard_t *shard) { } static size_t -pa_shard_ndirty(const pa_shard_t *shard) { +pac_sec_pinned_bytes_get(const sec_stats_t *stats) { + return min_zu(stats->bytes_pinned, stats->bytes); +} + +static size_t +pac_sec_dirty_npages_get(const sec_stats_t *stats) { + return (stats->bytes - pac_sec_pinned_bytes_get(stats)) >> LG_PAGE; +} + +static size_t +pa_shard_ndirty_no_pac_sec(const pa_shard_t *shard) { size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty); if (shard->ever_used_hpa) { ndirty += psset_ndirty(&shard->hpa.psset); @@ -90,6 +103,14 @@ pa_shard_ndirty(const pa_shard_t *shard) { return ndirty; } +static size_t +pa_shard_ndirty(const pa_shard_t *shard) { + sec_stats_t pac_sec_stats = {0}; + sec_stats_merge(TSDN_NULL, &shard->pac.sec, &pac_sec_stats); + return pa_shard_ndirty_no_pac_sec(shard) + + pac_sec_dirty_npages_get(&pac_sec_stats); +} + static size_t pa_shard_nmuzzy(const pa_shard_t *shard) { return ecache_npages_get(&shard->pac.ecache_muzzy); @@ -109,17 +130,29 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, hpa_shard_stats_t *hpa_stats_out, size_t *resident) { cassert(config_stats); + sec_stats_t pac_sec_stats = {0}; + sec_stats_merge(tsdn, &shard->pac.sec, &pac_sec_stats); + sec_stats_accum(&pa_shard_stats_out->pac_stats.pac_sec_stats, + &pac_sec_stats); + size_t pac_sec_pinned_bytes = pac_sec_pinned_bytes_get(&pac_sec_stats); + size_t pac_sec_pinned_npages = pac_sec_pinned_bytes >> LG_PAGE; + size_t pac_sec_dirty_npages = + pac_sec_dirty_npages_get(&pac_sec_stats); + pa_shard_stats_out->pac_stats.retained += ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE; pa_shard_stats_out->pac_stats.pinned += - ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE; + (ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE) + + pac_sec_pinned_bytes; pa_shard_stats_out->edata_avail += atomic_load_zu( &shard->edata_cache.count, ATOMIC_RELAXED); size_t resident_pgs = 0; resident_pgs += pa_shard_nactive(shard); - resident_pgs += pa_shard_ndirty(shard); + resident_pgs += pa_shard_ndirty_no_pac_sec(shard); + resident_pgs += pac_sec_dirty_npages; resident_pgs += ecache_npages_get(&shard->pac.ecache_pinned); + resident_pgs += pac_sec_pinned_npages; *resident += (resident_pgs << LG_PAGE); /* Dirty decay stats */ @@ -167,6 +200,16 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, pinned_bytes = ecache_nbytes_get( &shard->pac.ecache_pinned, i); + sec_pszind_stats_t pac_sec_pszind_stats = {0}; + sec_stats_merge_pszind( + tsdn, &shard->pac.sec, i, &pac_sec_pszind_stats); + dirty += pac_sec_pszind_stats.nextents + - pac_sec_pszind_stats.nextents_pinned; + dirty_bytes += pac_sec_pszind_stats.bytes + - pac_sec_pszind_stats.bytes_pinned; + pinned += pac_sec_pszind_stats.nextents_pinned; + pinned_bytes += pac_sec_pszind_stats.bytes_pinned; + estats_out[i].ndirty = dirty; estats_out[i].nmuzzy = muzzy; estats_out[i].nretained = retained; @@ -208,6 +251,9 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard, pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy); + sec_mutex_stats_read(tsdn, &shard->pac.sec, + &mutex_prof_data[arena_prof_mutex_pac_sec]); + if (shard->ever_used_hpa) { pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, &shard->hpa.mtx, arena_prof_mutex_hpa_shard); diff --git a/src/pac.c b/src/pac.c index 1b1c07b3..71c00062 100644 --- a/src/pac.c +++ b/src/pac.c @@ -7,6 +7,18 @@ #include "jemalloc/internal/san.h" #include "jemalloc/internal/witness.h" +static inline uint8_t +pac_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { + if (sec->opts.nshards <= 1) { + return 0; + } + if (tsdn_null(tsdn)) { + return 0; + } + tsd_t *tsd = tsdn_tsd(tsdn); + return sec_shard_pick(tsd, sec, tsd_pac_sec_shardp_get(tsd)); +} + static inline void pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay, pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) { @@ -95,6 +107,16 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, pac->stats_mtx = stats_mtx; atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED); + if (sec_init(tsdn, &pac->sec, base, &opt_pac_sec_opts)) { + /* sec_init already zeroed nshards and max_alloc. */ + } + if (!sec_is_used(&pac->sec) || dirty_decay_ms == 0) { + atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED); + } else { + atomic_store_zu(&pac->sec_max_alloc, + pac->sec.opts.max_alloc, ATOMIC_RELAXED); + } + return false; } @@ -136,6 +158,24 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, edata_t *edata = NULL; + if (!guarded && !zero && alignment <= PAGE + && size <= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) { + /* + * sec_max_alloc mirrors sec.opts.max_alloc when SEC is + * enabled, 0 when dirty decay is disabled. + */ + edata = sec_alloc(tsdn, &pac->sec, size, + pac_sec_shard_pick(tsdn, &pac->sec)); + if (edata != NULL) { + return edata; + } + /* + * Unlike HPA, PAC has no batch allocation primitive; we + * intentionally do not refill SEC on a miss. Extents enter the + * cache only through the dalloc path. + */ + } + /* * Guarded allocations need surrounding guard pages, which the pinned * pool does not maintain; skip ecache_pinned in that case. @@ -398,6 +438,37 @@ pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata, san_unguard_pages_two_sided( tsdn, ehooks, edata, pac->emap); } + } else if (edata_size_get(edata) + <= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) { + /* + * A dalloc can race with disabling SEC and cache an extent after + * the flush. Avoid a hot-path gate lock; such extents remain + * stats-tracked and are flushed by reset/destroy or a later + * disable. + */ + edata_list_active_t dalloc_list; + edata_list_active_init(&dalloc_list); + edata_list_active_append(&dalloc_list, edata); + sec_dalloc(tsdn, &pac->sec, &dalloc_list, + pac_sec_shard_pick(tsdn, &pac->sec)); + if (edata_list_active_empty(&dalloc_list)) { + *deferred_work_generated = false; + return; + } + /* Flush overflow extents to their backing ecaches. */ + bool any_deferred_work = false; + edata_t *flush_edata; + while ((flush_edata = + edata_list_active_first(&dalloc_list)) != NULL) { + edata_list_active_remove(&dalloc_list, + flush_edata); + if (!edata_pinned_get(flush_edata)) { + any_deferred_work = true; + } + pac_ecache_dalloc(tsdn, pac, ehooks, flush_edata); + } + *deferred_work_generated = any_deferred_work; + return; } bool pinned = edata_pinned_get(edata); @@ -720,6 +791,13 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, return true; } + bool update_pac_sec = (state == extent_state_dirty) + && sec_is_used(&pac->sec); + if (update_pac_sec && decay_ms == 0) { + atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED); + pac_sec_flush(tsdn, pac); + } + malloc_mutex_lock(tsdn, &decay->mtx); /* * Restart decay backlog from scratch, which may cause many dirty pages @@ -735,6 +813,11 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state, pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness); malloc_mutex_unlock(tsdn, &decay->mtx); + if (update_pac_sec && decay_ms != 0) { + atomic_store_zu(&pac->sec_max_alloc, + pac->sec.opts.max_alloc, ATOMIC_RELAXED); + } + return false; } @@ -809,3 +892,16 @@ pac_destroy(tsdn_t *tsdn, pac_t *pac) { extent_destroy_wrapper(tsdn, pac, ehooks, edata); } } + +void +pac_sec_flush(tsdn_t *tsdn, pac_t *pac) { + ehooks_t *ehooks = pac_ehooks_get(pac); + edata_list_active_t to_flush; + edata_list_active_init(&to_flush); + sec_flush(tsdn, &pac->sec, &to_flush); + edata_t *edata; + while ((edata = edata_list_active_first(&to_flush)) != NULL) { + edata_list_active_remove(&to_flush, edata); + pac_ecache_dalloc(tsdn, pac, ehooks, edata); + } +} diff --git a/src/sec.c b/src/sec.c index 6c15169e..b1b0e307 100644 --- a/src/sec.c +++ b/src/sec.c @@ -7,6 +7,7 @@ static bool sec_bin_init(sec_bin_t *bin) { atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED); + atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED); atomic_store_zu(&bin->ndalloc_flush, 0, ATOMIC_RELAXED); atomic_store_zu(&bin->nmisses, 0, ATOMIC_RELAXED); atomic_store_zu(&bin->nhits, 0, ATOMIC_RELAXED); @@ -24,8 +25,15 @@ sec_bin_init(sec_bin_t *bin) { bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) { + /* + * Invariant: max_alloc == 0 whenever nshards == 0. This lets + * sec_size_supported() collapse to a single comparison. + */ sec->opts = *opts; + sec->bins = NULL; + sec->npsizes = 0; if (opts->nshards == 0) { + sec->opts.max_alloc = 0; return false; } assert(opts->max_alloc >= PAGE); @@ -44,11 +52,15 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) { size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins; void *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE); if (dynalloc == NULL) { + sec->opts.nshards = 0; + sec->opts.max_alloc = 0; return true; } sec->bins = (sec_bin_t *)dynalloc; for (pszind_t j = 0; j < ntotal_bins; j++) { if (sec_bin_init(&sec->bins[j])) { + sec->opts.nshards = 0; + sec->opts.max_alloc = 0; return true; } } @@ -57,18 +69,16 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) { return false; } -static uint8_t -sec_shard_pick(tsdn_t *tsdn, sec_t *sec) { +uint8_t +sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp) { /* * Eventually, we should implement affinity, tracking source shard using * the edata_t's newly freed up fields. For now, just randomly * distribute across all shards. + * + * Callers must ensure sec->opts.nshards > 1. */ - if (tsdn_null(tsdn)) { - return 0; - } - tsd_t *tsd = tsdn_tsd(tsdn); - uint8_t *idxp = tsd_sec_shardp_get(tsd); + assert(sec->opts.nshards > 1); if (*idxp == (uint8_t)-1) { /* * First use; initialize using the trick from Daniel Lemire's @@ -135,6 +145,14 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) { size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED); assert(sz <= bytes_cur && sz > 0); bytes_cur -= sz; + if (edata_pinned_get(edata)) { + size_t bytes_pinned_cur = atomic_load_zu( + &bin->bytes_pinned_cur, ATOMIC_RELAXED); + assert(sz <= bytes_pinned_cur); + bytes_pinned_cur -= sz; + atomic_store_zu(&bin->bytes_pinned_cur, + bytes_pinned_cur, ATOMIC_RELAXED); + } atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED); atomic_load_add_store_zu(&bin->nhits, 1); } @@ -143,10 +161,10 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) { static edata_t * sec_multishard_trylock_alloc( - tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) { + tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind, uint8_t shard) { assert(sec->opts.nshards > 0); - uint8_t cur_shard = sec_shard_pick(tsdn, sec); + uint8_t cur_shard = shard; sec_bin_t *bin; for (size_t i = 0; i < sec->opts.nshards; ++i) { bin = sec_bin_pick(sec, cur_shard, pszind); @@ -170,7 +188,7 @@ sec_multishard_trylock_alloc( * declaring a miss. That could recover more remote-shard hits under * contention, but it also changes the allocation latency policy. */ - assert(cur_shard == sec_shard_pick(tsdn, sec)); + assert(cur_shard == shard); bin = sec_bin_pick(sec, cur_shard, pszind); malloc_mutex_lock(tsdn, &bin->mtx); edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size); @@ -184,10 +202,8 @@ sec_multishard_trylock_alloc( } edata_t * -sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) { - if (!sec_size_supported(sec, size)) { - return NULL; - } +sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard) { + assert(sec_size_supported(sec, size)); assert((size & PAGE_MASK) == 0); pszind_t pszind = sz_psz2ind(size); assert(pszind < sec->npsizes); @@ -208,7 +224,7 @@ sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) { /* frequent_reuse */ 1); return edata; } - return sec_multishard_trylock_alloc(tsdn, sec, size, pszind); + return sec_multishard_trylock_alloc(tsdn, sec, size, pszind, shard); } static void @@ -217,16 +233,23 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size, malloc_mutex_assert_owner(tsdn, &bin->mtx); size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED); + size_t bytes_pinned_cur = atomic_load_zu( + &bin->bytes_pinned_cur, ATOMIC_RELAXED); bytes_cur += size; edata_t *edata = edata_list_active_first(dalloc_list); assert(edata != NULL); edata_list_active_remove(dalloc_list, edata); JE_USDT(sec_dalloc, 3, sec, bin, edata); edata_list_active_prepend(&bin->freelist, edata); + if (edata_pinned_get(edata)) { + bytes_pinned_cur += size; + } /* Single extent can be returned to SEC */ assert(edata_list_active_empty(dalloc_list)); if (bytes_cur <= sec->opts.max_bytes) { + atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur, + ATOMIC_RELAXED); atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED); atomic_load_add_store_zu(&bin->ndalloc_noflush, 1); return; @@ -240,19 +263,25 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size, size_t sz = edata_size_get(cur); assert(sz <= bytes_cur && sz > 0); bytes_cur -= sz; + if (edata_pinned_get(cur)) { + assert(sz <= bytes_pinned_cur); + bytes_pinned_cur -= sz; + } edata_list_active_remove(&bin->freelist, cur); edata_list_active_append(dalloc_list, cur); } + atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur, + ATOMIC_RELAXED); atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED); } static void sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size, - pszind_t pszind, edata_list_active_t *dalloc_list) { + pszind_t pszind, edata_list_active_t *dalloc_list, uint8_t shard) { assert(sec->opts.nshards > 0); /* Try to dalloc in this threads bin first */ - uint8_t cur_shard = sec_shard_pick(tsdn, sec); + uint8_t cur_shard = shard; for (size_t i = 0; i < sec->opts.nshards; ++i) { sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind); if (!malloc_mutex_trylock(tsdn, &bin->mtx)) { @@ -267,7 +296,7 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size, } } /* No bin had alloc or had the extent */ - assert(cur_shard == sec_shard_pick(tsdn, sec)); + assert(cur_shard == shard); sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind); malloc_mutex_lock(tsdn, &bin->mtx); sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list); @@ -275,20 +304,16 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size, } void -sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) { - if (!sec_is_used(sec)) { - return; - } +sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list, + uint8_t shard) { edata_t *edata = edata_list_active_first(dalloc_list); size_t size = edata_size_get(edata); - if (size > sec->opts.max_alloc) { - return; - } + assert(sec_size_supported(sec, size)); pszind_t pszind = sz_psz2ind(size); assert(pszind < sec->npsizes); /* - * If there's only one shard, skip the trylock optimization and + * If there's only one shard, skip the trylock optimization and * go straight to the blocking lock. */ if (sec->opts.nshards == 1) { @@ -298,27 +323,49 @@ sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) { malloc_mutex_unlock(tsdn, &bin->mtx); return; } - sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list); + sec_multishard_trylock_dalloc( + tsdn, sec, size, pszind, dalloc_list, shard); +} + +static void +sec_list_pinned_bytes_get( + edata_list_active_t *list, size_t size, size_t *pinned_bytes) { + *pinned_bytes = 0; + for (edata_t *edata = edata_list_active_first(list); edata != NULL; + edata = edata_list_active_next(list, edata)) { + assert(edata_size_get(edata) == size); + if (edata_pinned_get(edata)) { + *pinned_bytes += size; + } + } } void sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result, - size_t nallocs) { + size_t nallocs, uint8_t shard) { assert((size & PAGE_MASK) == 0); - assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc); + assert(sec_size_supported(sec, size)); assert(nallocs > 0); pszind_t pszind = sz_psz2ind(size); assert(pszind < sec->npsizes); - sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind); + sec_bin_t *bin = sec_bin_pick(sec, shard, pszind); malloc_mutex_assert_not_owner(tsdn, &bin->mtx); malloc_mutex_lock(tsdn, &bin->mtx); - size_t new_cached_bytes = nallocs * size; size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED); + size_t bytes_pinned_cur = atomic_load_zu( + &bin->bytes_pinned_cur, ATOMIC_RELAXED); + size_t new_cached_bytes = nallocs * size; if (bytes_cur + new_cached_bytes <= sec->opts.max_bytes) { assert(!edata_list_active_empty(result)); + size_t new_cached_pinned_bytes; + sec_list_pinned_bytes_get( + result, size, &new_cached_pinned_bytes); + bytes_pinned_cur += new_cached_pinned_bytes; edata_list_active_concat(&bin->freelist, result); + atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur, + ATOMIC_RELAXED); atomic_store_zu(&bin->bytes_cur, bytes_cur + new_cached_bytes, ATOMIC_RELAXED); } else { @@ -336,7 +383,12 @@ sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result, assert(size == edata_size_get(edata)); edata_list_active_append(&bin->freelist, edata); bytes_cur += size; + if (edata_pinned_get(edata)) { + bytes_pinned_cur += size; + } } + atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur, + ATOMIC_RELAXED); atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED); } malloc_mutex_unlock(tsdn, &bin->mtx); @@ -352,21 +404,47 @@ sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) { sec_bin_t *bin = &sec->bins[i]; malloc_mutex_lock(tsdn, &bin->mtx); atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED); + atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED); edata_list_active_concat(to_flush, &bin->freelist); malloc_mutex_unlock(tsdn, &bin->mtx); } } +static void +sec_bin_bytes_get(const sec_bin_t *bin, size_t *bytes, size_t *bytes_pinned) { + *bytes = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED); + *bytes_pinned = atomic_load_zu( + &bin->bytes_pinned_cur, ATOMIC_RELAXED); + *bytes_pinned = min_zu(*bytes_pinned, *bytes); +} + +static void +sec_stats_merge_bin( + const sec_bin_t *bin, pszind_t pszind, sec_pszind_stats_t *stats) { + size_t bytes; + size_t bytes_pinned; + sec_bin_bytes_get(bin, &bytes, &bytes_pinned); + + stats->bytes += bytes; + stats->bytes_pinned += bytes_pinned; + size_t size = sz_pind2sz(pszind); + stats->nextents += bytes / size; + stats->nextents_pinned += bytes_pinned / size; +} + void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) { if (!sec_is_used(sec)) { return; } - size_t sum = 0; size_t ntotal_bins = sec->opts.nshards * sec->npsizes; for (pszind_t i = 0; i < ntotal_bins; i++) { sec_bin_t *bin = &sec->bins[i]; - sum += atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED); + size_t bytes; + size_t bytes_pinned; + sec_bin_bytes_get(bin, &bytes, &bytes_pinned); + stats->bytes += bytes; + stats->bytes_pinned += bytes_pinned; stats->total.nmisses += atomic_load_zu(&bin->nmisses, ATOMIC_RELAXED); stats->total.nhits += @@ -378,7 +456,20 @@ sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) { stats->total.noverfills += atomic_load_zu(&bin->noverfills, ATOMIC_RELAXED); } - stats->bytes += sum; +} + +void +sec_stats_merge_pszind(tsdn_t *tsdn, const sec_t *sec, pszind_t pszind, + sec_pszind_stats_t *stats) { + if (!sec_is_used(sec) || pszind >= sec->npsizes) { + return; + } + + for (size_t shard = 0; shard < sec->opts.nshards; shard++) { + size_t ind = shard * sec->npsizes + pszind; + assert(ind < sec->npsizes * sec->opts.nshards); + sec_stats_merge_bin(&sec->bins[ind], pszind, stats); + } } void diff --git a/src/stats.c b/src/stats.c index 2c741277..6e06bd55 100644 --- a/src/stats.c +++ b/src/stats.c @@ -838,6 +838,37 @@ stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) { &sec_overfills); } +static void +stats_arena_pac_sec_print(emitter_t *emitter, unsigned i) { + size_t sec_bytes; + size_t sec_hits; + size_t sec_misses; + size_t sec_dalloc_flush; + size_t sec_dalloc_noflush; + CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec_bytes, size_t); + emitter_kv(emitter, "pac_sec_bytes", + "Bytes in PAC small extent cache", + emitter_type_size, &sec_bytes); + CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec_hits, size_t); + emitter_kv(emitter, "pac_sec_hits", + "Total hits in PAC small extent cache", + emitter_type_size, &sec_hits); + CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec_misses, size_t); + emitter_kv(emitter, "pac_sec_misses", + "Total misses in PAC small extent cache", + emitter_type_size, &sec_misses); + CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i, + &sec_dalloc_noflush, size_t); + emitter_kv(emitter, "pac_sec_dalloc_noflush", + "Dalloc calls without flush in PAC small extent cache", + emitter_type_size, &sec_dalloc_noflush); + CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i, &sec_dalloc_flush, + size_t); + emitter_kv(emitter, "pac_sec_dalloc_flush", + "Dalloc calls with flush in PAC small extent cache", + emitter_type_size, &sec_dalloc_flush); +} + static void stats_arena_hpa_shard_counters_print( emitter_t *emitter, unsigned i, uint64_t uptime) { @@ -1570,6 +1601,10 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, GET_AND_EMIT_MEM_STAT(extent_avail) #undef GET_AND_EMIT_MEM_STAT + if (opt_pac_sec_opts.nshards > 0) { + stats_arena_pac_sec_print(emitter, i); + } + if (mutex) { stats_arena_mutexes_print(emitter, i, uptime); } @@ -1764,6 +1799,9 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_SIZE_T("hpa_sec_nshards") OPT_WRITE_SIZE_T("hpa_sec_max_alloc") OPT_WRITE_SIZE_T("hpa_sec_max_bytes") + OPT_WRITE_SIZE_T("experimental_pac_sec_nshards") + OPT_WRITE_SIZE_T("experimental_pac_sec_max_alloc") + OPT_WRITE_SIZE_T("experimental_pac_sec_max_bytes") OPT_WRITE_BOOL("huge_arena_pac_thp") OPT_WRITE_CHAR_P("metadata_thp") OPT_WRITE_INT64("mutex_max_spin") diff --git a/test/unit/json_stats.c b/test/unit/json_stats.c index c206974b..11b397d9 100644 --- a/test/unit/json_stats.c +++ b/test/unit/json_stats.c @@ -181,7 +181,7 @@ static const size_t num_global_mutexes = sizeof(global_mutex_names) static const char *arena_mutex_names[] = {"large", "extent_avail", "extents_dirty", "extents_muzzy", "extents_retained", "decay_dirty", "decay_muzzy", "base", "tcache_list", "hpa_shard", "hpa_shard_grow", - "hpa_sec"}; + "hpa_sec", "pac_sec"}; static const size_t num_arena_mutexes = sizeof(arena_mutex_names) / sizeof(arena_mutex_names[0]); diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 031641a5..c91ec08a 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -530,6 +530,9 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always); TEST_MALLCTL_OPT(size_t, hpa_sec_max_alloc, always); TEST_MALLCTL_OPT(size_t, hpa_sec_max_bytes, always); + TEST_MALLCTL_OPT(size_t, experimental_pac_sec_nshards, always); + TEST_MALLCTL_OPT(size_t, experimental_pac_sec_max_alloc, always); + TEST_MALLCTL_OPT(size_t, experimental_pac_sec_max_bytes, always); TEST_MALLCTL_OPT(ssize_t, experimental_hpa_max_purge_nhp, always); TEST_MALLCTL_OPT(size_t, hpa_purge_threshold, always); TEST_MALLCTL_OPT(uint64_t, hpa_min_purge_delay_ms, always); diff --git a/test/unit/pac_sec_integration.c b/test/unit/pac_sec_integration.c new file mode 100644 index 00000000..be8915d9 --- /dev/null +++ b/test/unit/pac_sec_integration.c @@ -0,0 +1,387 @@ +#include "test/jemalloc_test.h" + +/* + * Use multiple shards while keeping the single-threaded shard choice + * deterministic. Disable background threads to avoid decay races. + */ +const char *malloc_conf = + "experimental_pac_sec_nshards:2,background_thread:false"; + +static sec_opts_t saved_pac_sec_opts; + +static void +pac_sec_test_opts_set(void) { + saved_pac_sec_opts = opt_pac_sec_opts; + /* + * Include cache-oblivious padding so PAC can cache the test size. + */ + size_t test_extent_size = SC_LARGE_MINCLASS + sz_large_pad; + opt_pac_sec_opts.max_alloc = test_extent_size; + opt_pac_sec_opts.max_bytes = 4 * test_extent_size; +} + +static void +pac_sec_test_opts_restore(void) { + opt_pac_sec_opts = saved_pac_sec_opts; +} + +static void * +pinned_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, + size_t size, size_t alignment, bool *zero, bool *commit, + unsigned arena_ind) { + void *ret = ehooks_default_extent_hooks.alloc( + (extent_hooks_t *)&ehooks_default_extent_hooks, new_addr, size, + alignment, zero, commit, arena_ind); + if (ret == NULL) { + return NULL; + } + if (!*commit) { + if (ehooks_default_extent_hooks.commit != NULL + && ehooks_default_extent_hooks.commit( + (extent_hooks_t *)&ehooks_default_extent_hooks, ret, + size, 0, size, arena_ind)) { + ehooks_default_extent_hooks.dalloc( + (extent_hooks_t *)&ehooks_default_extent_hooks, ret, + size, *commit, arena_ind); + return NULL; + } + *commit = true; + } + return (void *)((uintptr_t)ret | EXTENT_ALLOC_FLAG_PINNED); +} + +static void +pinned_extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, + bool committed, unsigned arena_ind) { + ehooks_default_extent_hooks.destroy( + (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, + committed, arena_ind); +} + +static bool +pinned_extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { + return ehooks_default_extent_hooks.split( + (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, size_a, + size_b, committed, arena_ind); +} + +static bool +pinned_extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { + return ehooks_default_extent_hooks.merge( + (extent_hooks_t *)&ehooks_default_extent_hooks, addr_a, size_a, + addr_b, size_b, committed, arena_ind); +} + +static extent_hooks_t pinned_hooks = { + pinned_extent_alloc, + NULL, /* dalloc */ + pinned_extent_destroy, + NULL, /* commit */ + NULL, /* decommit */ + NULL, /* purge_lazy */ + NULL, /* purge_forced */ + pinned_extent_split, + pinned_extent_merge +}; + +static size_t +read_arena_stat(unsigned arena_ind, const char *field) { + char cmd[128]; + size_t val; + size_t sz = sizeof(val); + uint64_t epoch = 1; + sz = sizeof(epoch); + expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0, + "Unexpected mallctl failure"); + sz = sizeof(val); + snprintf(cmd, sizeof(cmd), "stats.arenas.%u.%s", arena_ind, field); + expect_d_eq(mallctl(cmd, (void *)&val, &sz, NULL, 0), 0, + "Unexpected mallctl failure reading arena stat"); + return val; +} + +static size_t +read_stat(unsigned arena_ind, const char *field) { + char cmd[128]; + snprintf(cmd, sizeof(cmd), "pac_sec_%s", field); + return read_arena_stat(arena_ind, cmd); +} + +static size_t +read_extents_stat(unsigned arena_ind, pszind_t pszind, const char *field) { + char cmd[128]; + snprintf(cmd, sizeof(cmd), "extents.%u.%s", pszind, field); + return read_arena_stat(arena_ind, cmd); +} + +static size_t +read_pinned_npages(unsigned arena_ind) { + tsd_t *tsd = tsd_fetch(); + arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false); + expect_ptr_not_null(arena, "arena_get failed"); + return ecache_npages_get(&arena->pa_shard.pac.ecache_pinned); +} + +static void +dirty_decay_ms_set(unsigned arena_ind, ssize_t decay_ms) { + char cmd[64]; + snprintf(cmd, sizeof(cmd), "arena.%u.dirty_decay_ms", arena_ind); + expect_d_eq(mallctl(cmd, NULL, NULL, (void *)&decay_ms, + sizeof(decay_ms)), 0, "dirty_decay_ms mallctl failed"); +} + +TEST_BEGIN(test_pac_sec_alloc_dalloc_cycle) { + test_skip_if(!config_stats); + test_skip_if(opt_hpa); + + pac_sec_test_opts_set(); + unsigned arena_ind; + size_t sz = sizeof(arena_ind); + expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), + 0, "Unexpected arenas.create failure"); + + int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; + size_t alloc_size = SC_LARGE_MINCLASS; + + size_t max_bytes; + sz = sizeof(max_bytes); + expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes", + (void *)&max_bytes, &sz, NULL, 0), 0, + "Unexpected mallctl failure"); + + size_t capacity = max_bytes / alloc_size; + expect_zu_gt(capacity, 0, "SEC capacity must be > 0 for this test"); + + /* Initial allocation misses SEC. */ + void *p1 = mallocx(alloc_size, flags); + expect_ptr_not_null(p1, "mallocx failed"); + size_t resident_after_alloc = read_arena_stat(arena_ind, "resident"); + expect_zu_eq(read_stat(arena_ind, "misses"), 1, + "first alloc should miss SEC"); + expect_zu_eq(read_stat(arena_ind, "hits"), 0, + "no hits yet"); + expect_zu_eq(read_stat(arena_ind, "bytes"), 0, + "SEC should be empty (extent is active)"); + + /* Freeing the extent caches it in PAC SEC. */ + dallocx(p1, flags); + size_t cached_after_one = read_stat(arena_ind, "bytes"); + expect_zu_gt(cached_after_one, 0, + "SEC should cache the freed extent"); + expect_zu_ge(read_arena_stat(arena_ind, "pdirty"), + cached_after_one >> LG_PAGE, + "SEC bytes should count toward dirty page stats"); + expect_zu_ge(read_arena_stat(arena_ind, "resident"), + resident_after_alloc, + "SEC bytes should remain included in resident stats"); + /* Use the actual extent size, including size class rounding. */ + size_t extent_size = cached_after_one; + pszind_t extent_pszind = sz_psz2ind(extent_size); + expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "ndirty"), 1, + "SEC extents should count toward extents dirty stats"); + expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "dirty_bytes"), + cached_after_one, + "SEC bytes should count toward extents dirty byte stats"); + expect_zu_eq(read_stat(arena_ind, "dalloc_noflush"), 1, + "one dalloc absorbed without flush"); + expect_zu_eq(read_stat(arena_ind, "dalloc_flush"), 0, + "no flush yet"); + + /* Recompute capacity based on actual extent size. */ + capacity = max_bytes / extent_size; + expect_zu_gt(capacity, 0, "SEC capacity should be positive"); + + /* Reallocate from SEC. */ + void *p2 = mallocx(alloc_size, flags); + expect_ptr_not_null(p2, "mallocx failed"); + expect_zu_eq(read_stat(arena_ind, "hits"), 1, + "second alloc should hit SEC"); + expect_zu_eq(read_stat(arena_ind, "misses"), 1, + "misses should not increase"); + expect_zu_eq(read_stat(arena_ind, "bytes"), 0, + "SEC should be empty after hit"); + + dallocx(p2, flags); + + /* Overflow SEC and flush cold extents to ecache_dirty. */ + size_t nallocs = capacity + 2; + void **ptrs = mallocx(nallocs * sizeof(void *), + MALLOCX_TCACHE_NONE); + expect_ptr_not_null(ptrs, "metadata alloc failed"); + + for (size_t i = 0; i < nallocs; i++) { + ptrs[i] = mallocx(alloc_size, flags); + expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i); + } + for (size_t i = 0; i < nallocs; i++) { + dallocx(ptrs[i], flags); + } + + size_t noflush = read_stat(arena_ind, "dalloc_noflush"); + size_t flush = read_stat(arena_ind, "dalloc_flush"); + size_t cached_bytes = read_stat(arena_ind, "bytes"); + + expect_zu_gt(noflush, 1, + "most dallocs should be absorbed"); + expect_zu_gt(flush, 0, + "overflow should trigger at least one flush"); + expect_zu_gt(cached_bytes, 0, + "SEC should still hold extents after partial flush"); + expect_zu_le(cached_bytes, max_bytes, + "SEC should not exceed max_bytes"); + + /* A populated SEC serves the next allocation. */ + size_t misses_before = read_stat(arena_ind, "misses"); + void *p3 = mallocx(alloc_size, flags); + expect_ptr_not_null(p3, "mallocx failed"); + expect_zu_eq(read_stat(arena_ind, "misses"), misses_before, + "alloc from populated SEC should not miss"); + dallocx(p3, flags); + + /* Purge flushes SEC entirely. */ + char cmd[64]; + snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind); + expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, + "purge failed"); + expect_zu_eq(read_stat(arena_ind, "bytes"), 0, + "SEC should be empty after purge"); + + /* Allocation after purge misses SEC. */ + size_t hits_before = read_stat(arena_ind, "hits"); + void *p4 = mallocx(alloc_size, flags); + expect_ptr_not_null(p4, "mallocx failed"); + expect_zu_eq(read_stat(arena_ind, "hits"), hits_before, + "alloc after purge should miss SEC"); + dallocx(p4, flags); + + dallocx(ptrs, MALLOCX_TCACHE_NONE); + snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind); + expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, + "arena destroy failed"); + pac_sec_test_opts_restore(); +} +TEST_END + +TEST_BEGIN(test_pac_sec_dirty_decay_toggle) { + test_skip_if(!config_stats); + test_skip_if(opt_hpa); + + pac_sec_test_opts_set(); + unsigned arena_ind; + size_t sz = sizeof(arena_ind); + expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0), + 0, "Unexpected arenas.create failure"); + + int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; + size_t alloc_size = SC_LARGE_MINCLASS; + + void *p = mallocx(alloc_size, flags); + expect_ptr_not_null(p, "mallocx failed"); + dallocx(p, flags); + expect_zu_gt(read_stat(arena_ind, "bytes"), 0, + "SEC should cache when dirty decay is enabled"); + + dirty_decay_ms_set(arena_ind, 0); + expect_zu_eq(read_stat(arena_ind, "bytes"), 0, + "disabling dirty decay should flush SEC"); + + p = mallocx(alloc_size, flags); + expect_ptr_not_null(p, "mallocx failed"); + dallocx(p, flags); + expect_zu_eq(read_stat(arena_ind, "bytes"), 0, + "SEC should stay disabled while dirty decay is zero"); + + dirty_decay_ms_set(arena_ind, 100); + p = mallocx(alloc_size, flags); + expect_ptr_not_null(p, "mallocx failed"); + dallocx(p, flags); + expect_zu_gt(read_stat(arena_ind, "bytes"), 0, + "SEC should be usable after dirty decay is re-enabled"); + + char cmd[64]; + snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind); + expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, + "arena destroy failed"); + pac_sec_test_opts_restore(); +} +TEST_END + +TEST_BEGIN(test_pac_sec_flush_pinned) { + test_skip_if(!config_stats); + test_skip_if(opt_hpa); + + pac_sec_test_opts_set(); + unsigned arena_ind; + size_t sz = sizeof(arena_ind); + extent_hooks_t *hooks_ptr = &pinned_hooks; + expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, + &hooks_ptr, sizeof(hooks_ptr)), 0, + "Unexpected arenas.create failure"); + + int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; + size_t alloc_size = SC_LARGE_MINCLASS; + size_t max_bytes; + sz = sizeof(max_bytes); + expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes", + (void *)&max_bytes, &sz, NULL, 0), 0, + "Unexpected mallctl failure"); + + void *p = mallocx(alloc_size, flags); + expect_ptr_not_null(p, "mallocx failed"); + size_t resident_after_alloc = read_arena_stat(arena_ind, "resident"); + dallocx(p, flags); + size_t sec_bytes = read_stat(arena_ind, "bytes"); + expect_zu_gt(sec_bytes, 0, "SEC should cache the pinned extent"); + expect_zu_ge(read_arena_stat(arena_ind, "pinned"), sec_bytes, + "Pinned bytes cached in SEC should count toward pinned stats"); + expect_zu_ge(read_arena_stat(arena_ind, "resident"), + resident_after_alloc, + "Pinned SEC bytes should remain included in resident stats"); + + size_t extent_size = sec_bytes; + pszind_t extent_pszind = sz_psz2ind(extent_size); + expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "npinned"), 1, + "Pinned SEC extents should count toward extents pinned stats"); + expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "pinned_bytes"), + sec_bytes, + "Pinned SEC bytes should count toward extents pinned byte stats"); + size_t nallocs = max_bytes / extent_size + 2; + void **ptrs = mallocx(nallocs * sizeof(void *), MALLOCX_TCACHE_NONE); + expect_ptr_not_null(ptrs, "metadata alloc failed"); + for (size_t i = 0; i < nallocs; i++) { + ptrs[i] = mallocx(alloc_size, flags); + expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i); + } + size_t pinned_before_overflow = read_pinned_npages(arena_ind); + for (size_t i = 0; i < nallocs; i++) { + dallocx(ptrs[i], flags); + } + expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_overflow, + "SEC overflow should flush pinned extents to ecache_pinned"); + + size_t pinned_before_purge = read_pinned_npages(arena_ind); + char cmd[64]; + snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind); + expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, + "purge failed"); + expect_zu_eq(read_stat(arena_ind, "bytes"), 0, + "SEC should be empty after purge"); + expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_purge, + "PAC SEC purge should flush pinned extents to ecache_pinned"); + + dallocx(ptrs, MALLOCX_TCACHE_NONE); + snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind); + expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0, + "arena destroy failed"); + pac_sec_test_opts_restore(); +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_pac_sec_alloc_dalloc_cycle, test_pac_sec_dirty_decay_toggle, + test_pac_sec_flush_pinned); +} diff --git a/test/unit/sec.c b/test/unit/sec.c index 8caf0a6a..c73aa28a 100644 --- a/test/unit/sec.c +++ b/test/unit/sec.c @@ -12,6 +12,17 @@ struct test_data_s { base_t *base; }; +static void +edata_init_test(edata_t *edata) { + memset(edata, 0, sizeof(*edata)); +} + +static void +edata_init_test_size(edata_t *edata, size_t size) { + edata_init_test(edata); + edata_size_set(edata, size); +} + static void test_data_init(tsdn_t *tsdn, test_data_t *tdata, const sec_opts_t *opts) { tdata->base = base_new(TSDN_NULL, /* ind */ 123, @@ -31,6 +42,32 @@ destroy_test_data(tsdn_t *tsdn, test_data_t *tdata) { base_delete(tsdn, tdata->base); } +static uint8_t +test_sec_shard(tsdn_t *tsdn, sec_t *sec) { + if (tsdn_null(tsdn) || sec->opts.nshards <= 1) { + return 0; + } + tsd_t *tsd = tsdn_tsd(tsdn); + return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd)); +} + +static edata_t * +sec_test_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) { + return sec_alloc(tsdn, sec, size, test_sec_shard(tsdn, sec)); +} + +static void +sec_test_fill(tsdn_t *tsdn, sec_t *sec, size_t size, + edata_list_active_t *result, size_t nallocs) { + sec_fill(tsdn, sec, size, result, nallocs, test_sec_shard(tsdn, sec)); +} + +static void +sec_test_dalloc(tsdn_t *tsdn, sec_t *sec, + edata_list_active_t *dalloc_list) { + sec_dalloc(tsdn, sec, dalloc_list, test_sec_shard(tsdn, sec)); +} + TEST_BEGIN(test_max_nshards_option_zero) { test_data_t tdata; sec_opts_t opts; @@ -41,7 +78,8 @@ TEST_BEGIN(test_max_nshards_option_zero) { tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); test_data_init(tsdn, &tdata, &opts); - edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE); + edata_t *edata = sec_size_supported(&tdata.sec, PAGE) + ? sec_test_alloc(tsdn, &tdata.sec, PAGE) : NULL; expect_ptr_null(edata, "SEC should be disabled when nshards==0"); destroy_test_data(tsdn, &tdata); } @@ -57,7 +95,8 @@ TEST_BEGIN(test_max_alloc_option_too_small) { tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); test_data_init(tsdn, &tdata, &opts); - edata_t *edata = sec_alloc(tsdn, &tdata.sec, 3 * PAGE); + edata_t *edata = sec_size_supported(&tdata.sec, 3 * PAGE) + ? sec_test_alloc(tsdn, &tdata.sec, 3 * PAGE) : NULL; expect_ptr_null(edata, "max_alloc is 2*PAGE, should not alloc 3*PAGE"); destroy_test_data(tsdn, &tdata); } @@ -78,11 +117,11 @@ TEST_BEGIN(test_sec_fill) { edata_list_active_t allocs; edata_list_active_init(&allocs); edata_t edata1, edata2; - edata_size_set(&edata1, PAGE); - edata_size_set(&edata2, PAGE); + edata_init_test_size(&edata1, PAGE); + edata_init_test_size(&edata2, PAGE); edata_list_active_append(&allocs, &edata1); edata_list_active_append(&allocs, &edata2); - sec_fill(tsdn, &tdata.sec, PAGE, &allocs, 2); + sec_test_fill(tsdn, &tdata.sec, PAGE, &allocs, 2); sec_stats_merge(tsdn, &tdata.sec, &stats); expect_zu_eq(stats.bytes, 2 * PAGE, "SEC should have what we filled"); expect_true(edata_list_active_empty(&allocs), @@ -91,13 +130,13 @@ TEST_BEGIN(test_sec_fill) { /* Try to overfill and confirm that max_bytes is respected. */ stats.bytes = 0; edata_t edata5, edata4, edata3; - edata_size_set(&edata3, PAGE); - edata_size_set(&edata4, PAGE); - edata_size_set(&edata5, PAGE); + edata_init_test_size(&edata3, PAGE); + edata_init_test_size(&edata4, PAGE); + edata_init_test_size(&edata5, PAGE); edata_list_active_append(&allocs, &edata3); edata_list_active_append(&allocs, &edata4); edata_list_active_append(&allocs, &edata5); - sec_fill(tsdn, &tdata.sec, PAGE, &allocs, 3); + sec_test_fill(tsdn, &tdata.sec, PAGE, &allocs, 3); sec_stats_merge(tsdn, &tdata.sec, &stats); expect_zu_eq( stats.bytes, opts.max_bytes, "SEC can't have more than max_bytes"); @@ -118,20 +157,20 @@ TEST_BEGIN(test_sec_alloc) { test_data_init(tsdn, &tdata, &opts); /* Alloc from empty cache returns NULL */ - edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE); + edata_t *edata = sec_test_alloc(tsdn, &tdata.sec, PAGE); expect_ptr_null(edata, "SEC is empty"); /* Place two extents into the sec */ edata_list_active_t allocs; edata_list_active_init(&allocs); edata_t edata1, edata2; - edata_size_set(&edata1, PAGE); + edata_init_test_size(&edata1, PAGE); edata_list_active_append(&allocs, &edata1); - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_true(edata_list_active_empty(&allocs), ""); - edata_size_set(&edata2, PAGE); + edata_init_test_size(&edata2, PAGE); edata_list_active_append(&allocs, &edata2); - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_true(edata_list_active_empty(&allocs), ""); sec_stats_t stats = {0}; @@ -141,20 +180,20 @@ TEST_BEGIN(test_sec_alloc) { stats.bytes = 0; /* Most recently cached extent should be used on alloc */ - edata = sec_alloc(tsdn, &tdata.sec, PAGE); + edata = sec_test_alloc(tsdn, &tdata.sec, PAGE); expect_ptr_eq(edata, &edata2, "edata2 is most recently used"); sec_stats_merge(tsdn, &tdata.sec, &stats); expect_zu_eq(stats.bytes, PAGE, "One more item left in the cache"); stats.bytes = 0; /* Alloc can still get extents from cache */ - edata = sec_alloc(tsdn, &tdata.sec, PAGE); + edata = sec_test_alloc(tsdn, &tdata.sec, PAGE); expect_ptr_eq(edata, &edata1, "SEC is not empty"); sec_stats_merge(tsdn, &tdata.sec, &stats); expect_zu_eq(stats.bytes, 0, "No more items after last one is popped"); /* And cache is empty again */ - edata = sec_alloc(tsdn, &tdata.sec, PAGE); + edata = sec_test_alloc(tsdn, &tdata.sec, PAGE); expect_ptr_null(edata, "SEC is empty"); destroy_test_data(tsdn, &tdata); } @@ -174,20 +213,20 @@ TEST_BEGIN(test_sec_dalloc) { edata_list_active_t allocs; edata_list_active_init(&allocs); edata_t edata1; - edata_size_set(&edata1, PAGE); + edata_init_test_size(&edata1, PAGE); edata_list_active_append(&allocs, &edata1); /* SEC is empty, we return one pointer to it */ - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_true( edata_list_active_empty(&allocs), "extents should be consumed"); /* Return one more extent, so that we are at the limit */ edata_t edata2; - edata_size_set(&edata2, PAGE); + edata_init_test_size(&edata2, PAGE); edata_list_active_append(&allocs, &edata2); /* Sec can take one more as well and we will be exactly at max_bytes */ - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_true( edata_list_active_empty(&allocs), "extents should be consumed"); @@ -203,9 +242,9 @@ TEST_BEGIN(test_sec_dalloc) { * the one given in the input as it is the most recently used. */ edata_t edata3; - edata_size_set(&edata3, PAGE); + edata_init_test_size(&edata3, PAGE); edata_list_active_append(&allocs, &edata3); - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_false( edata_list_active_empty(&allocs), "extents should NOT be consumed"); expect_ptr_ne( @@ -218,6 +257,65 @@ TEST_BEGIN(test_sec_dalloc) { } TEST_END +TEST_BEGIN(test_sec_pinned_stats) { + test_data_t tdata; + sec_opts_t opts; + opts.nshards = 1; + opts.max_alloc = PAGE; + opts.max_bytes = 2 * PAGE; + + tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); + test_data_init(tsdn, &tdata, &opts); + + edata_list_active_t allocs; + edata_list_active_init(&allocs); + edata_t pinned, unpinned; + edata_init_test_size(&pinned, PAGE); + edata_pinned_set(&pinned, true); + edata_init_test_size(&unpinned, PAGE); + + edata_list_active_append(&allocs, &pinned); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); + edata_list_active_append(&allocs, &unpinned); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); + + sec_stats_t stats = {0}; + sec_stats_merge(tsdn, &tdata.sec, &stats); + expect_zu_eq(stats.bytes, 2 * PAGE, ""); + expect_zu_eq(stats.bytes_pinned, PAGE, ""); + sec_pszind_stats_t pszind_stats = {0}; + sec_stats_merge_pszind( + tsdn, &tdata.sec, sz_psz2ind(PAGE), &pszind_stats); + expect_zu_eq(pszind_stats.nextents, 2, ""); + expect_zu_eq(pszind_stats.bytes, 2 * PAGE, ""); + expect_zu_eq(pszind_stats.nextents_pinned, 1, ""); + expect_zu_eq(pszind_stats.bytes_pinned, PAGE, ""); + + memset(&stats, 0, sizeof(stats)); + expect_ptr_eq(sec_test_alloc(tsdn, &tdata.sec, PAGE), &unpinned, + "Unpinned extent is the most recently cached"); + sec_stats_merge(tsdn, &tdata.sec, &stats); + expect_zu_eq(stats.bytes, PAGE, ""); + expect_zu_eq(stats.bytes_pinned, PAGE, ""); + memset(&pszind_stats, 0, sizeof(pszind_stats)); + sec_stats_merge_pszind( + tsdn, &tdata.sec, sz_psz2ind(PAGE), &pszind_stats); + expect_zu_eq(pszind_stats.nextents, 1, ""); + expect_zu_eq(pszind_stats.bytes, PAGE, ""); + expect_zu_eq(pszind_stats.nextents_pinned, 1, ""); + expect_zu_eq(pszind_stats.bytes_pinned, PAGE, ""); + + memset(&stats, 0, sizeof(stats)); + expect_ptr_eq(sec_test_alloc(tsdn, &tdata.sec, PAGE), &pinned, + "Pinned extent should be tracked until allocated"); + sec_stats_merge(tsdn, &tdata.sec, &stats); + expect_zu_eq(stats.bytes, 0, ""); + expect_zu_eq(stats.bytes_pinned, 0, ""); + + destroy_test_data(tsdn, &tdata); +} +TEST_END + TEST_BEGIN(test_max_bytes_too_low) { test_data_t tdata; sec_opts_t opts; @@ -232,11 +330,11 @@ TEST_BEGIN(test_max_bytes_too_low) { edata_list_active_t allocs; edata_list_active_init(&allocs); edata_t edata1; - edata_size_set(&edata1, 3 * PAGE); + edata_init_test_size(&edata1, 3 * PAGE); edata_list_active_append(&allocs, &edata1); /* SEC is empty, we return one pointer to it */ - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_false( edata_list_active_empty(&allocs), "extents should not be consumed"); destroy_test_data(tsdn, &tdata); @@ -262,13 +360,13 @@ TEST_BEGIN(test_sec_flush) { edata_t edata1[NALLOCS]; edata_t edata4[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edata_size_set(&edata1[i], PAGE); - edata_size_set(&edata4[i], 4 * PAGE); + edata_init_test_size(&edata1[i], PAGE); + edata_init_test_size(&edata4[i], 4 * PAGE); edata_list_active_append(&allocs1, &edata1[i]); - sec_dalloc(tsdn, &tdata.sec, &allocs1); + sec_test_dalloc(tsdn, &tdata.sec, &allocs1); edata_list_active_append(&allocs4, &edata4[i]); - sec_dalloc(tsdn, &tdata.sec, &allocs4); + sec_test_dalloc(tsdn, &tdata.sec, &allocs4); } sec_stats_t stats = {0}; @@ -301,23 +399,23 @@ TEST_BEGIN(test_sec_stats) { edata_list_active_t allocs; edata_list_active_init(&allocs); edata_t edata1; - edata_size_set(&edata1, PAGE); + edata_init_test_size(&edata1, PAGE); edata_list_active_append(&allocs, &edata1); /* SEC is empty alloc fails. nmisses==1 */ - edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE); + edata_t *edata = sec_test_alloc(tsdn, &tdata.sec, PAGE); expect_ptr_null(edata, "SEC should be empty"); /* SEC is empty, we return one pointer to it. ndalloc_noflush=1 */ - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_true( edata_list_active_empty(&allocs), "extents should be consumed"); edata_t edata2; - edata_size_set(&edata2, PAGE); + edata_init_test_size(&edata2, PAGE); edata_list_active_append(&allocs, &edata2); /* Sec can take one more, so ndalloc_noflush=2 */ - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_true( edata_list_active_empty(&allocs), "extents should be consumed"); @@ -335,9 +433,9 @@ TEST_BEGIN(test_sec_stats) { * the limit. This will force flush, so ndalloc_flush = 1. */ edata_t edata3; - edata_size_set(&edata3, PAGE); + edata_init_test_size(&edata3, PAGE); edata_list_active_append(&allocs, &edata3); - sec_dalloc(tsdn, &tdata.sec, &allocs); + sec_test_dalloc(tsdn, &tdata.sec, &allocs); expect_false( edata_list_active_empty(&allocs), "extents should NOT be consumed"); sec_stats_merge(tsdn, &tdata.sec, &stats); @@ -351,11 +449,6 @@ TEST_END #define NOPS_PER_THREAD 100 #define NPREFILL 32 -static void -edata_init_test(edata_t *edata) { - memset(edata, 0, sizeof(*edata)); -} - typedef struct { sec_t *sec; uint8_t preferred_shard; @@ -379,12 +472,12 @@ thd_trylock_test(void *varg) { *shard_idx = arg->preferred_shard; /* Fill the shard with some extents */ - sec_fill(tsdn, arg->sec, PAGE, &arg->fill_list, arg->fill_list_sz); + sec_test_fill(tsdn, arg->sec, PAGE, &arg->fill_list, arg->fill_list_sz); expect_true(edata_list_active_empty(&arg->fill_list), ""); for (unsigned i = 0; i < NOPS_PER_THREAD; i++) { /* Try to allocate from SEC */ - arg->edata[i] = sec_alloc(tsdn, arg->sec, PAGE); + arg->edata[i] = sec_test_alloc(tsdn, arg->sec, PAGE); if (arg->edata[i] != NULL) { expect_zu_eq(edata_size_get(arg->edata[i]), PAGE, ""); } @@ -397,7 +490,7 @@ thd_trylock_test(void *varg) { arg->nallocs++; edata_list_active_append(&list, arg->edata[i]); expect_zu_eq(edata_size_get(arg->edata[i]), PAGE, ""); - sec_dalloc(tsdn, arg->sec, &list); + sec_test_dalloc(tsdn, arg->sec, &list); if (edata_list_active_empty(&list)) { arg->ndallocs++; } else { @@ -433,8 +526,7 @@ TEST_BEGIN(test_sec_multishard) { edata_list_active_init(&args[i].fill_list); for (unsigned j = 0; j < NPREFILL; ++j) { size_t ind = i * NPREFILL + j; - edata_init_test(&all_edatas[ind]); - edata_size_set(&all_edatas[ind], PAGE); + edata_init_test_size(&all_edatas[ind], PAGE); edata_list_active_append( &args[i].fill_list, &all_edatas[ind]); } @@ -487,6 +579,6 @@ int main(void) { return test(test_max_nshards_option_zero, test_max_alloc_option_too_small, test_sec_fill, test_sec_alloc, - test_sec_dalloc, test_max_bytes_too_low, test_sec_flush, - test_sec_stats, test_sec_multishard); + test_sec_dalloc, test_sec_pinned_stats, test_max_bytes_too_low, + test_sec_flush, test_sec_stats, test_sec_multishard); }