Use SEC in PAC to reduce lock contention on the ecaches

Add a small extent cache in front of the PAC ecaches. Allocs and dallocs
that fit are served from per-shard SEC bins without taking the ecache
mutex; overflow falls through to the backing ecaches, including
ecache_pinned for pinned extents.

The feature is gated behind experimental_pac_sec_nshards (default 0,
disabled). To support independent HPA and PAC SEC instances,
sec_alloc/sec_dalloc/sec_fill take an explicit shard argument, with HPA
and PAC using separate TSD shard slots.

Makefile.in

@@ -272,6 +272,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/oversize_threshold.c \
 	$(srcroot)test/unit/pa.c \
 	$(srcroot)test/unit/pac.c \
+	$(srcroot)test/unit/pac_sec_integration.c \
 	$(srcroot)test/unit/pack.c \
 	$(srcroot)test/unit/pages.c \
 	$(srcroot)test/unit/peak.c \

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -21,6 +21,7 @@ extern bool             opt_confirm_conf;
 extern bool             opt_hpa;
 extern hpa_shard_opts_t opt_hpa_opts;
 extern sec_opts_t       opt_hpa_sec_opts;
+extern sec_opts_t       opt_pac_sec_opts;
 
 extern const char *opt_junk;
 extern bool        opt_junk_alloc;

include/jemalloc/internal/mutex_prof.h

@@ -37,7 +37,8 @@ typedef enum {
 	OP(tcache_list)                                                        \
 	OP(hpa_shard)                                                          \
 	OP(hpa_shard_grow)                                                     \
-	OP(hpa_sec)
+	OP(hpa_sec)                                                            \
+	OP(pac_sec)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,

include/jemalloc/internal/pac.h

@@ -8,6 +8,7 @@
 #include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
 #include "jemalloc/internal/lockedint.h"
+#include "jemalloc/internal/sec.h"
 #include "jemalloc/internal/tsd_types.h"
 #include "san_bump.h"
 
@@ -84,12 +85,24 @@ struct pac_stats_s {
 
 	/* VM space had to be leaked (undocumented).  Normally 0. */
 	atomic_zu_t abandoned_vm;
+
+	/* PAC SEC stats.  Derived. */
+	sec_stats_t pac_sec_stats;
 };
 
 typedef struct pac_s pac_t;
 struct pac_s {
+	/* Small extent cache in front of PAC ecaches to reduce contention. */
+	sec_t sec;
+	/*
+	 * Runtime gate for PAC SEC.  0 disables (when SEC is not configured or
+	 * dirty_decay_ms == 0); otherwise mirrors sec.opts.max_alloc.
+	 */
+	atomic_zu_t sec_max_alloc;
+
 	/* True once pinned memory has been seen. */
 	atomic_b_t has_pinned;
+
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.
@@ -236,4 +249,6 @@ ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
 void pac_destroy(tsdn_t *tsdn, pac_t *pac);
 
+void pac_sec_flush(tsdn_t *tsdn, pac_t *pac);
+
 #endif /* JEMALLOC_INTERNAL_PAC_H */

include/jemalloc/internal/sec.h

@@ -35,11 +35,21 @@ typedef struct sec_stats_s sec_stats_t;
 struct sec_stats_s {
 	/* Sum of bytes_cur across all shards. */
 	size_t bytes;
+	/* Subset of bytes that are pinned. */
+	size_t bytes_pinned;
 
 	/* Totals of bin_stats. */
 	sec_bin_stats_t total;
 };
 
+typedef struct sec_pszind_stats_s sec_pszind_stats_t;
+struct sec_pszind_stats_s {
+	size_t nextents;
+	size_t bytes;
+	size_t nextents_pinned;
+	size_t bytes_pinned;
+};
+
 static inline void
 sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
 	dst->nmisses += src->nmisses;
@@ -52,6 +62,7 @@ sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
 static inline void
 sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 	dst->bytes += src->bytes;
+	dst->bytes_pinned += src->bytes_pinned;
 	sec_bin_stats_accum(&dst->total, &src->total);
 }
 
@@ -67,6 +78,7 @@ struct sec_bin_s {
 	 * Number of bytes in this particular bin.
 	 */
 	atomic_zu_t         bytes_cur;
+	atomic_zu_t         bytes_pinned_cur;
 	edata_list_active_t freelist;
 	atomic_zu_t         nmisses;
 	atomic_zu_t         nhits;
@@ -88,8 +100,8 @@ sec_is_used(const sec_t *sec) {
 }
 
 static inline bool
-sec_size_supported(sec_t *sec, size_t size) {
+sec_size_supported(const sec_t *sec, size_t size) {
-	return sec_is_used(sec) && size <= sec->opts.max_alloc;
+	return size <= sec->opts.max_alloc;
 }
 
 /* Min number of extents we will allocate when expanding the SEC. */
@@ -111,12 +123,21 @@ sec_size_supported(sec_t *sec, size_t size) {
 void sec_calc_nallocs_for_size(
     sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
 
-/* If sec does not have extent available, it will return NULL. */
+/*
-edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
+ * Lazily picks (and caches in *idxp) a shard for the calling thread.  Different
+ * SEC instances pass independent per-thread uint8_t slots, initialized to
+ * (uint8_t)-1.
+ */
+uint8_t sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp);
+
+/* Callers must ensure sec_size_supported(sec, size). */
+edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard);
 void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
-        edata_list_active_t *result, size_t nallocs);
+        edata_list_active_t *result, size_t nallocs, uint8_t shard);
 
 /*
+ * Callers must ensure sec_size_supported(sec, edata_size).
+ *
  * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty
  * if there are extents that need to be flushed from cache.  Please note, that
  * if we need to flush, extent(s) returned in the list to be deallocated
@@ -124,7 +145,8 @@ void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
  * considered "hot" and preserved in the cache, while "colder" ones are
  * returned).
  */
-void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list);
+void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
+    uint8_t shard);
 
 bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts);
 
@@ -138,6 +160,9 @@ void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
  * split), which simplifies the stats management.
  */
 void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge_pszind(
+    tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
+    sec_pszind_stats_t *stats);
 void sec_mutex_stats_read(
     tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data);
 

include/jemalloc/internal/tsd_internals.h

@@ -83,6 +83,7 @@ typedef void (*test_callback_t)(int *);
 	O(arena, arena_t *, arena_t *)                                         \
 	O(arena_decay_ticker, ticker_geom_t, ticker_geom_t)                    \
 	O(sec_shard, uint8_t, uint8_t)                                         \
+	O(pac_sec_shard, uint8_t, uint8_t)                                     \
 	O(binshards, tsd_binshards_t, tsd_binshards_t)                         \
 	O(peak, peak_t, peak_t)                                                \
 	O(tcache_slow, tcache_slow_t, tcache_slow_t)                           \
@@ -102,6 +103,7 @@ typedef void (*test_callback_t)(int *);
 	    /* arena */ NULL, /* arena_decay_ticker */                         \
 	    TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),                   \
 	    /* sec_shard */ (uint8_t) - 1,                                     \
+	    /* pac_sec_shard */ (uint8_t) - 1,                                 \
 	    /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER,                    \
 	    /* peak */ PEAK_INITIALIZER, /* tcache_slow */                     \
 	    TCACHE_SLOW_ZERO_INITIALIZER,                                      \

src/conf.c

@@ -950,6 +950,17 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
 			    "hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
 			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
+			CONF_HANDLE_SIZE_T(opt_pac_sec_opts.nshards,
+			    "experimental_pac_sec_nshards", 0, 0,
+			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
+			CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_alloc,
+			    "experimental_pac_sec_max_alloc", PAGE,
+			    USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN,
+			    CONF_CHECK_MAX, true);
+			CONF_HANDLE_SIZE_T(opt_pac_sec_opts.max_bytes,
+			    "experimental_pac_sec_max_bytes",
+			    SEC_OPTS_MAX_BYTES_DEFAULT, 0,
+			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
 
 			if (CONF_MATCH("slab_sizes")) {
 				if (CONF_MATCH_VALUE("default")) {

src/ctl.c

@@ -147,6 +147,9 @@ CTL_PROTO(opt_hpa_dirty_mult)
 CTL_PROTO(opt_hpa_sec_nshards)
 CTL_PROTO(opt_hpa_sec_max_alloc)
 CTL_PROTO(opt_hpa_sec_max_bytes)
+CTL_PROTO(opt_experimental_pac_sec_nshards)
+CTL_PROTO(opt_experimental_pac_sec_max_alloc)
+CTL_PROTO(opt_experimental_pac_sec_max_bytes)
 CTL_PROTO(opt_huge_arena_pac_thp)
 CTL_PROTO(opt_metadata_thp)
 CTL_PROTO(opt_retain)
@@ -382,6 +385,11 @@ CTL_PROTO(stats_arenas_i_hpa_sec_misses)
 CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_flush)
 CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_noflush)
 CTL_PROTO(stats_arenas_i_hpa_sec_overfills)
+CTL_PROTO(stats_arenas_i_pac_sec_bytes)
+CTL_PROTO(stats_arenas_i_pac_sec_hits)
+CTL_PROTO(stats_arenas_i_pac_sec_misses)
+CTL_PROTO(stats_arenas_i_pac_sec_dalloc_flush)
+CTL_PROTO(stats_arenas_i_pac_sec_dalloc_noflush)
 INDEX_PROTO(stats_arenas_i)
 CTL_PROTO(stats_allocated)
 CTL_PROTO(stats_active)
@@ -520,6 +528,12 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
     {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
     {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
     {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
+    {NAME("experimental_pac_sec_nshards"),
+        CTL(opt_experimental_pac_sec_nshards)},
+    {NAME("experimental_pac_sec_max_alloc"),
+        CTL(opt_experimental_pac_sec_max_alloc)},
+    {NAME("experimental_pac_sec_max_bytes"),
+        CTL(opt_experimental_pac_sec_max_bytes)},
     {NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
     {NAME("metadata_thp"), CTL(opt_metadata_thp)},
     {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)},
@@ -883,6 +897,12 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
         CTL(stats_arenas_i_hpa_sec_dalloc_noflush)},
     {NAME("hpa_sec_dalloc_flush"), CTL(stats_arenas_i_hpa_sec_dalloc_flush)},
     {NAME("hpa_sec_overfills"), CTL(stats_arenas_i_hpa_sec_overfills)},
+    {NAME("pac_sec_bytes"), CTL(stats_arenas_i_pac_sec_bytes)},
+    {NAME("pac_sec_hits"), CTL(stats_arenas_i_pac_sec_hits)},
+    {NAME("pac_sec_misses"), CTL(stats_arenas_i_pac_sec_misses)},
+    {NAME("pac_sec_dalloc_noflush"),
+        CTL(stats_arenas_i_pac_sec_dalloc_noflush)},
+    {NAME("pac_sec_dalloc_flush"), CTL(stats_arenas_i_pac_sec_dalloc_flush)},
     {NAME("small"), CHILD(named, stats_arenas_i_small)},
     {NAME("large"), CHILD(named, stats_arenas_i_large)},
     {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
@@ -1269,6 +1289,10 @@ ctl_arena_stats_sdmerge(
 		    &sdstats->astats.pa_shard_stats.pac_stats.abandoned_vm,
 		    &astats->astats.pa_shard_stats.pac_stats.abandoned_vm);
 
+		sec_stats_accum(
+		    &sdstats->astats.pa_shard_stats.pac_stats.pac_sec_stats,
+		    &astats->astats.pa_shard_stats.pac_stats.pac_sec_stats);
+
 		sdstats->astats.tcache_bytes += astats->astats.tcache_bytes;
 		sdstats->astats.tcache_stashed_bytes +=
 		    astats->astats.tcache_stashed_bytes;
@@ -2235,6 +2259,12 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
+CTL_RO_NL_GEN(opt_experimental_pac_sec_nshards,
+    opt_pac_sec_opts.nshards, size_t)
+CTL_RO_NL_GEN(opt_experimental_pac_sec_max_alloc,
+    opt_pac_sec_opts.max_alloc, size_t)
+CTL_RO_NL_GEN(opt_experimental_pac_sec_max_bytes,
+    opt_pac_sec_opts.max_bytes, size_t)
 CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
 CTL_RO_NL_GEN(
     opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
@@ -3829,6 +3859,27 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_noflush,
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_overfills,
     arenas_i(mib[2])->astats->hpastats.secstats.total.noverfills, size_t)
 
+CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_bytes,
+    arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
+	.pac_sec_stats.bytes,
+    size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_hits,
+    arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
+	.pac_sec_stats.total.nhits,
+    size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_misses,
+    arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
+	.pac_sec_stats.total.nmisses,
+    size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_dalloc_flush,
+    arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
+	.pac_sec_stats.total.ndalloc_flush,
+    size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_pac_sec_dalloc_noflush,
+    arenas_i(mib[2])->astats->astats.pa_shard_stats.pac_stats
+	.pac_sec_stats.total.ndalloc_noflush,
+    size_t)
+
 CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
     arenas_i(mib[2])->astats->allocated_small, size_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc,

src/hpa.c

@@ -14,6 +14,18 @@ const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
 bool opt_experimental_hpa_start_huge_if_thp_always = true;
 bool opt_experimental_hpa_enforce_hugify = false;
 
+static inline uint8_t
+hpa_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
+	if (sec->opts.nshards <= 1) {
+		return 0;
+	}
+	if (tsdn_null(tsdn)) {
+		return 0;
+	}
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd));
+}
+
 JET_EXTERN bool
 hpa_hugepage_size_exceeds_limit(void) {
 	return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE;
@@ -945,10 +957,14 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	    && (size > shard->opts.slab_max_alloc)) {
 		return NULL;
 	}
-	edata_t *edata = sec_alloc(tsdn, &shard->sec, size);
+	edata_t *edata = NULL;
+	if (sec_size_supported(&shard->sec, size)) {
+		edata = sec_alloc(tsdn, &shard->sec, size,
+		    hpa_sec_shard_pick(tsdn, &shard->sec));
 		if (edata != NULL) {
 			return edata;
 		}
+	}
 	edata_list_active_t results;
 	edata_list_active_init(&results);
 	size_t min_nallocs, max_nallocs;
@@ -966,7 +982,8 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
 	}
 	if (nsuccess > 0) {
 		assert(sec_size_supported(&shard->sec, size));
-		sec_fill(tsdn, &shard->sec, size, &results, nsuccess);
+		sec_fill(tsdn, &shard->sec, size, &results, nsuccess,
+		    hpa_sec_shard_pick(tsdn, &shard->sec));
 		/* Unlikely rollback in case of overfill */
 		if (!edata_list_active_empty(&results)) {
 			hpa_dalloc_batch(
@@ -1059,12 +1076,15 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
 	edata_list_active_init(&dalloc_list);
 	edata_list_active_append(&dalloc_list, edata);
 
-	sec_dalloc(tsdn, &shard->sec, &dalloc_list);
+	if (sec_size_supported(&shard->sec, edata_size_get(edata))) {
+		sec_dalloc(tsdn, &shard->sec, &dalloc_list,
+		    hpa_sec_shard_pick(tsdn, &shard->sec));
 		if (edata_list_active_empty(&dalloc_list)) {
 			/* sec consumed the pointer */
 			*deferred_work_generated = false;
 			return;
 		}
+	}
 	/* We may have more than one pointer to flush now */
 	hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated);
 }

src/jemalloc.c

@@ -198,6 +198,9 @@ size_t opt_calloc_madvise_threshold = CALLOC_MADVISE_THRESHOLD_DEFAULT;
 bool             opt_hpa = false;
 hpa_shard_opts_t opt_hpa_opts = HPA_SHARD_OPTS_DEFAULT;
 sec_opts_t       opt_hpa_sec_opts = SEC_OPTS_DEFAULT;
+sec_opts_t       opt_pac_sec_opts = {0,
+    (32 * 1024) > (PAGE * 2) ? (32 * 1024) : (PAGE * 2),
+    SEC_OPTS_MAX_BYTES_DEFAULT};
 
 /* False should be the common case.  Set to true to trigger initialization. */
 bool malloc_slow = true;

src/pa.c

@@ -94,6 +94,7 @@ pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
 
 void
 pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
+	pac_sec_flush(tsdn, &shard->pac);
 	if (shard->ever_used_hpa) {
 		hpa_shard_flush(tsdn, &shard->hpa);
 	}

src/pa_extra.c

@@ -17,6 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
 
 void
 pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
+	sec_prefork2(tsdn, &shard->pac.sec);
 	if (shard->ever_used_hpa) {
 		hpa_shard_prefork2(tsdn, &shard->hpa);
 	}
@@ -54,6 +55,7 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_retained);
 	ecache_postfork_parent(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.grow_mtx);
+	sec_postfork_parent(tsdn, &shard->pac.sec);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
@@ -69,6 +71,7 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
 	ecache_postfork_child(tsdn, &shard->pac.ecache_retained);
 	ecache_postfork_child(tsdn, &shard->pac.ecache_pinned);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.grow_mtx);
+	sec_postfork_child(tsdn, &shard->pac.sec);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
@@ -82,7 +85,17 @@ pa_shard_nactive(const pa_shard_t *shard) {
 }
 
 static size_t
-pa_shard_ndirty(const pa_shard_t *shard) {
+pac_sec_pinned_bytes_get(const sec_stats_t *stats) {
+	return min_zu(stats->bytes_pinned, stats->bytes);
+}
+
+static size_t
+pac_sec_dirty_npages_get(const sec_stats_t *stats) {
+	return (stats->bytes - pac_sec_pinned_bytes_get(stats)) >> LG_PAGE;
+}
+
+static size_t
+pa_shard_ndirty_no_pac_sec(const pa_shard_t *shard) {
 	size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty);
 	if (shard->ever_used_hpa) {
 		ndirty += psset_ndirty(&shard->hpa.psset);
@@ -90,6 +103,14 @@ pa_shard_ndirty(const pa_shard_t *shard) {
 	return ndirty;
 }
 
+static size_t
+pa_shard_ndirty(const pa_shard_t *shard) {
+	sec_stats_t pac_sec_stats = {0};
+	sec_stats_merge(TSDN_NULL, &shard->pac.sec, &pac_sec_stats);
+	return pa_shard_ndirty_no_pac_sec(shard)
+	    + pac_sec_dirty_npages_get(&pac_sec_stats);
+}
+
 static size_t
 pa_shard_nmuzzy(const pa_shard_t *shard) {
 	return ecache_npages_get(&shard->pac.ecache_muzzy);
@@ -109,17 +130,29 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     hpa_shard_stats_t *hpa_stats_out, size_t *resident) {
 	cassert(config_stats);
 
+	sec_stats_t pac_sec_stats = {0};
+	sec_stats_merge(tsdn, &shard->pac.sec, &pac_sec_stats);
+	sec_stats_accum(&pa_shard_stats_out->pac_stats.pac_sec_stats,
+	    &pac_sec_stats);
+	size_t pac_sec_pinned_bytes = pac_sec_pinned_bytes_get(&pac_sec_stats);
+	size_t pac_sec_pinned_npages = pac_sec_pinned_bytes >> LG_PAGE;
+	size_t pac_sec_dirty_npages =
+	    pac_sec_dirty_npages_get(&pac_sec_stats);
+
 	pa_shard_stats_out->pac_stats.retained +=
 	    ecache_npages_get(&shard->pac.ecache_retained) << LG_PAGE;
 	pa_shard_stats_out->pac_stats.pinned +=
-	    ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE;
+	    (ecache_npages_get(&shard->pac.ecache_pinned) << LG_PAGE)
+	    + pac_sec_pinned_bytes;
 	pa_shard_stats_out->edata_avail += atomic_load_zu(
 	    &shard->edata_cache.count, ATOMIC_RELAXED);
 
 	size_t resident_pgs = 0;
 	resident_pgs += pa_shard_nactive(shard);
-	resident_pgs += pa_shard_ndirty(shard);
+	resident_pgs += pa_shard_ndirty_no_pac_sec(shard);
+	resident_pgs += pac_sec_dirty_npages;
 	resident_pgs += ecache_npages_get(&shard->pac.ecache_pinned);
+	resident_pgs += pac_sec_pinned_npages;
 	*resident += (resident_pgs << LG_PAGE);
 
 	/* Dirty decay stats */
@@ -167,6 +200,16 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 		pinned_bytes = ecache_nbytes_get(
 		    &shard->pac.ecache_pinned, i);
 
+		sec_pszind_stats_t pac_sec_pszind_stats = {0};
+		sec_stats_merge_pszind(
+		    tsdn, &shard->pac.sec, i, &pac_sec_pszind_stats);
+		dirty += pac_sec_pszind_stats.nextents
+		    - pac_sec_pszind_stats.nextents_pinned;
+		dirty_bytes += pac_sec_pszind_stats.bytes
+		    - pac_sec_pszind_stats.bytes_pinned;
+		pinned += pac_sec_pszind_stats.nextents_pinned;
+		pinned_bytes += pac_sec_pszind_stats.bytes_pinned;
+
 		estats_out[i].ndirty = dirty;
 		estats_out[i].nmuzzy = muzzy;
 		estats_out[i].nretained = retained;
@@ -208,6 +251,9 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 	pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 	    &shard->pac.decay_muzzy.mtx, arena_prof_mutex_decay_muzzy);
 
+	sec_mutex_stats_read(tsdn, &shard->pac.sec,
+	    &mutex_prof_data[arena_prof_mutex_pac_sec]);
+
 	if (shard->ever_used_hpa) {
 		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 		    &shard->hpa.mtx, arena_prof_mutex_hpa_shard);

src/pac.c

@@ -7,6 +7,18 @@
 #include "jemalloc/internal/san.h"
 #include "jemalloc/internal/witness.h"
 
+static inline uint8_t
+pac_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
+	if (sec->opts.nshards <= 1) {
+		return 0;
+	}
+	if (tsdn_null(tsdn)) {
+		return 0;
+	}
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	return sec_shard_pick(tsd, sec, tsd_pac_sec_shardp_get(tsd));
+}
+
 static inline void
 pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay,
     pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) {
@@ -95,6 +107,16 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	pac->stats_mtx = stats_mtx;
 	atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
 
+	if (sec_init(tsdn, &pac->sec, base, &opt_pac_sec_opts)) {
+		/* sec_init already zeroed nshards and max_alloc. */
+	}
+	if (!sec_is_used(&pac->sec) || dirty_decay_ms == 0) {
+		atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED);
+	} else {
+		atomic_store_zu(&pac->sec_max_alloc,
+		    pac->sec.opts.max_alloc, ATOMIC_RELAXED);
+	}
+
 	return false;
 }
 
@@ -136,6 +158,24 @@ pac_alloc_real(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size,
 
 	edata_t *edata = NULL;
 
+	if (!guarded && !zero && alignment <= PAGE
+	    && size <= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) {
+		/*
+		 * sec_max_alloc mirrors sec.opts.max_alloc when SEC is
+		 * enabled, 0 when dirty decay is disabled.
+		 */
+		edata = sec_alloc(tsdn, &pac->sec, size,
+		    pac_sec_shard_pick(tsdn, &pac->sec));
+		if (edata != NULL) {
+			return edata;
+		}
+		/*
+		 * Unlike HPA, PAC has no batch allocation primitive; we
+		 * intentionally do not refill SEC on a miss.  Extents enter the
+		 * cache only through the dalloc path.
+		 */
+	}
+
 	/*
 	 * Guarded allocations need surrounding guard pages, which the pinned
 	 * pool does not maintain; skip ecache_pinned in that case.
@@ -398,6 +438,37 @@ pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
 			san_unguard_pages_two_sided(
 			    tsdn, ehooks, edata, pac->emap);
 		}
+	} else if (edata_size_get(edata)
+	    <= atomic_load_zu(&pac->sec_max_alloc, ATOMIC_RELAXED)) {
+		/*
+		 * A dalloc can race with disabling SEC and cache an extent after
+		 * the flush.  Avoid a hot-path gate lock; such extents remain
+		 * stats-tracked and are flushed by reset/destroy or a later
+		 * disable.
+		 */
+		edata_list_active_t dalloc_list;
+		edata_list_active_init(&dalloc_list);
+		edata_list_active_append(&dalloc_list, edata);
+		sec_dalloc(tsdn, &pac->sec, &dalloc_list,
+		    pac_sec_shard_pick(tsdn, &pac->sec));
+		if (edata_list_active_empty(&dalloc_list)) {
+			*deferred_work_generated = false;
+			return;
+		}
+		/* Flush overflow extents to their backing ecaches. */
+		bool any_deferred_work = false;
+		edata_t *flush_edata;
+		while ((flush_edata =
+		    edata_list_active_first(&dalloc_list)) != NULL) {
+			edata_list_active_remove(&dalloc_list,
+			    flush_edata);
+			if (!edata_pinned_get(flush_edata)) {
+				any_deferred_work = true;
+			}
+			pac_ecache_dalloc(tsdn, pac, ehooks, flush_edata);
+		}
+		*deferred_work_generated = any_deferred_work;
+		return;
 	}
 
 	bool pinned = edata_pinned_get(edata);
@@ -720,6 +791,13 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
 		return true;
 	}
 
+	bool update_pac_sec = (state == extent_state_dirty)
+	    && sec_is_used(&pac->sec);
+	if (update_pac_sec && decay_ms == 0) {
+		atomic_store_zu(&pac->sec_max_alloc, 0, ATOMIC_RELAXED);
+		pac_sec_flush(tsdn, pac);
+	}
+
 	malloc_mutex_lock(tsdn, &decay->mtx);
 	/*
 	 * Restart decay backlog from scratch, which may cause many dirty pages
@@ -735,6 +813,11 @@ pac_decay_ms_set(tsdn_t *tsdn, pac_t *pac, extent_state_t state,
 	pac_maybe_decay_purge(tsdn, pac, decay, decay_stats, ecache, eagerness);
 	malloc_mutex_unlock(tsdn, &decay->mtx);
 
+	if (update_pac_sec && decay_ms != 0) {
+		atomic_store_zu(&pac->sec_max_alloc,
+		    pac->sec.opts.max_alloc, ATOMIC_RELAXED);
+	}
+
 	return false;
 }
 
@@ -809,3 +892,16 @@ pac_destroy(tsdn_t *tsdn, pac_t *pac) {
 		extent_destroy_wrapper(tsdn, pac, ehooks, edata);
 	}
 }
+
+void
+pac_sec_flush(tsdn_t *tsdn, pac_t *pac) {
+	ehooks_t *ehooks = pac_ehooks_get(pac);
+	edata_list_active_t to_flush;
+	edata_list_active_init(&to_flush);
+	sec_flush(tsdn, &pac->sec, &to_flush);
+	edata_t *edata;
+	while ((edata = edata_list_active_first(&to_flush)) != NULL) {
+		edata_list_active_remove(&to_flush, edata);
+		pac_ecache_dalloc(tsdn, pac, ehooks, edata);
+	}
+}

src/sec.c

@@ -7,6 +7,7 @@
 static bool
 sec_bin_init(sec_bin_t *bin) {
 	atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
+	atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
 	atomic_store_zu(&bin->ndalloc_flush, 0, ATOMIC_RELAXED);
 	atomic_store_zu(&bin->nmisses, 0, ATOMIC_RELAXED);
 	atomic_store_zu(&bin->nhits, 0, ATOMIC_RELAXED);
@@ -24,8 +25,15 @@ sec_bin_init(sec_bin_t *bin) {
 
 bool
 sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
+	/*
+	 * Invariant: max_alloc == 0 whenever nshards == 0.  This lets
+	 * sec_size_supported() collapse to a single comparison.
+	 */
 	sec->opts = *opts;
+	sec->bins = NULL;
+	sec->npsizes = 0;
 	if (opts->nshards == 0) {
+		sec->opts.max_alloc = 0;
 		return false;
 	}
 	assert(opts->max_alloc >= PAGE);
@@ -44,11 +52,15 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
 	size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
 	void  *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
 	if (dynalloc == NULL) {
+		sec->opts.nshards = 0;
+		sec->opts.max_alloc = 0;
 		return true;
 	}
 	sec->bins = (sec_bin_t *)dynalloc;
 	for (pszind_t j = 0; j < ntotal_bins; j++) {
 		if (sec_bin_init(&sec->bins[j])) {
+			sec->opts.nshards = 0;
+			sec->opts.max_alloc = 0;
 			return true;
 		}
 	}
@@ -57,18 +69,16 @@ sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
 	return false;
 }
 
-static uint8_t
+uint8_t
-sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
+sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp) {
 	/*
 	 * Eventually, we should implement affinity, tracking source shard using
 	 * the edata_t's newly freed up fields.  For now, just randomly
 	 * distribute across all shards.
+	 *
+	 * Callers must ensure sec->opts.nshards > 1.
 	 */
-	if (tsdn_null(tsdn)) {
+	assert(sec->opts.nshards > 1);
-		return 0;
-	}
-	tsd_t   *tsd = tsdn_tsd(tsdn);
-	uint8_t *idxp = tsd_sec_shardp_get(tsd);
 	if (*idxp == (uint8_t)-1) {
 		/*
 		 * First use; initialize using the trick from Daniel Lemire's
@@ -135,6 +145,14 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
 		size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
 		assert(sz <= bytes_cur && sz > 0);
 		bytes_cur -= sz;
+		if (edata_pinned_get(edata)) {
+			size_t bytes_pinned_cur = atomic_load_zu(
+			    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
+			assert(sz <= bytes_pinned_cur);
+			bytes_pinned_cur -= sz;
+			atomic_store_zu(&bin->bytes_pinned_cur,
+			    bytes_pinned_cur, ATOMIC_RELAXED);
+		}
 		atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
 		atomic_load_add_store_zu(&bin->nhits, 1);
 	}
@@ -143,10 +161,10 @@ sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
 
 static edata_t *
 sec_multishard_trylock_alloc(
-    tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
+    tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind, uint8_t shard) {
 	assert(sec->opts.nshards > 0);
 
-	uint8_t    cur_shard = sec_shard_pick(tsdn, sec);
+	uint8_t    cur_shard = shard;
 	sec_bin_t *bin;
 	for (size_t i = 0; i < sec->opts.nshards; ++i) {
 		bin = sec_bin_pick(sec, cur_shard, pszind);
@@ -170,7 +188,7 @@ sec_multishard_trylock_alloc(
 	 * declaring a miss.  That could recover more remote-shard hits under
 	 * contention, but it also changes the allocation latency policy.
 	 */
-	assert(cur_shard == sec_shard_pick(tsdn, sec));
+	assert(cur_shard == shard);
 	bin = sec_bin_pick(sec, cur_shard, pszind);
 	malloc_mutex_lock(tsdn, &bin->mtx);
 	edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
@@ -184,10 +202,8 @@ sec_multishard_trylock_alloc(
 }
 
 edata_t *
-sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
+sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard) {
-	if (!sec_size_supported(sec, size)) {
+	assert(sec_size_supported(sec, size));
-		return NULL;
-	}
 	assert((size & PAGE_MASK) == 0);
 	pszind_t pszind = sz_psz2ind(size);
 	assert(pszind < sec->npsizes);
@@ -208,7 +224,7 @@ sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
 		    /* frequent_reuse */ 1);
 		return edata;
 	}
-	return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
+	return sec_multishard_trylock_alloc(tsdn, sec, size, pszind, shard);
 }
 
 static void
@@ -217,16 +233,23 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
 	malloc_mutex_assert_owner(tsdn, &bin->mtx);
 
 	size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+	size_t bytes_pinned_cur = atomic_load_zu(
+	    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
 	bytes_cur += size;
 	edata_t *edata = edata_list_active_first(dalloc_list);
 	assert(edata != NULL);
 	edata_list_active_remove(dalloc_list, edata);
 	JE_USDT(sec_dalloc, 3, sec, bin, edata);
 	edata_list_active_prepend(&bin->freelist, edata);
+	if (edata_pinned_get(edata)) {
+		bytes_pinned_cur += size;
+	}
 	/* Single extent can be returned to SEC */
 	assert(edata_list_active_empty(dalloc_list));
 
 	if (bytes_cur <= sec->opts.max_bytes) {
+		atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+		    ATOMIC_RELAXED);
 		atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
 		atomic_load_add_store_zu(&bin->ndalloc_noflush, 1);
 		return;
@@ -240,19 +263,25 @@ sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
 		size_t   sz = edata_size_get(cur);
 		assert(sz <= bytes_cur && sz > 0);
 		bytes_cur -= sz;
+		if (edata_pinned_get(cur)) {
+			assert(sz <= bytes_pinned_cur);
+			bytes_pinned_cur -= sz;
+		}
 		edata_list_active_remove(&bin->freelist, cur);
 		edata_list_active_append(dalloc_list, cur);
 	}
+	atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+	    ATOMIC_RELAXED);
 	atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
 }
 
 static void
 sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
-    pszind_t pszind, edata_list_active_t *dalloc_list) {
+    pszind_t pszind, edata_list_active_t *dalloc_list, uint8_t shard) {
 	assert(sec->opts.nshards > 0);
 
 	/* Try to dalloc in this threads bin first */
-	uint8_t cur_shard = sec_shard_pick(tsdn, sec);
+	uint8_t cur_shard = shard;
 	for (size_t i = 0; i < sec->opts.nshards; ++i) {
 		sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
 		if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
@@ -267,7 +296,7 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
 		}
 	}
 	/* No bin had alloc or had the extent */
-	assert(cur_shard == sec_shard_pick(tsdn, sec));
+	assert(cur_shard == shard);
 	sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
 	malloc_mutex_lock(tsdn, &bin->mtx);
 	sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
@@ -275,15 +304,11 @@ sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
 }
 
 void
-sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
+sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
-	if (!sec_is_used(sec)) {
+    uint8_t shard) {
-		return;
-	}
 	edata_t *edata = edata_list_active_first(dalloc_list);
 	size_t   size = edata_size_get(edata);
-	if (size > sec->opts.max_alloc) {
+	assert(sec_size_supported(sec, size));
-		return;
-	}
 	pszind_t pszind = sz_psz2ind(size);
 	assert(pszind < sec->npsizes);
 
@@ -298,27 +323,49 @@ sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
 		malloc_mutex_unlock(tsdn, &bin->mtx);
 		return;
 	}
-	sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
+	sec_multishard_trylock_dalloc(
+	    tsdn, sec, size, pszind, dalloc_list, shard);
+}
+
+static void
+sec_list_pinned_bytes_get(
+    edata_list_active_t *list, size_t size, size_t *pinned_bytes) {
+	*pinned_bytes = 0;
+	for (edata_t *edata = edata_list_active_first(list); edata != NULL;
+	    edata = edata_list_active_next(list, edata)) {
+		assert(edata_size_get(edata) == size);
+		if (edata_pinned_get(edata)) {
+			*pinned_bytes += size;
+		}
+	}
 }
 
 void
 sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
-    size_t nallocs) {
+    size_t nallocs, uint8_t shard) {
 	assert((size & PAGE_MASK) == 0);
-	assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
+	assert(sec_size_supported(sec, size));
 	assert(nallocs > 0);
 
 	pszind_t pszind = sz_psz2ind(size);
 	assert(pszind < sec->npsizes);
 
-	sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
+	sec_bin_t *bin = sec_bin_pick(sec, shard, pszind);
 	malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
 	malloc_mutex_lock(tsdn, &bin->mtx);
-	size_t new_cached_bytes = nallocs * size;
 	size_t bytes_cur = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+	size_t bytes_pinned_cur = atomic_load_zu(
+	    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
+	size_t new_cached_bytes = nallocs * size;
 	if (bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
 		assert(!edata_list_active_empty(result));
+		size_t new_cached_pinned_bytes;
+		sec_list_pinned_bytes_get(
+		    result, size, &new_cached_pinned_bytes);
+		bytes_pinned_cur += new_cached_pinned_bytes;
 		edata_list_active_concat(&bin->freelist, result);
+		atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+		    ATOMIC_RELAXED);
 		atomic_store_zu(&bin->bytes_cur, bytes_cur + new_cached_bytes,
 		    ATOMIC_RELAXED);
 	} else {
@@ -336,7 +383,12 @@ sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
 			assert(size == edata_size_get(edata));
 			edata_list_active_append(&bin->freelist, edata);
 			bytes_cur += size;
+			if (edata_pinned_get(edata)) {
+				bytes_pinned_cur += size;
 			}
+		}
+		atomic_store_zu(&bin->bytes_pinned_cur, bytes_pinned_cur,
+		    ATOMIC_RELAXED);
 		atomic_store_zu(&bin->bytes_cur, bytes_cur, ATOMIC_RELAXED);
 	}
 	malloc_mutex_unlock(tsdn, &bin->mtx);
@@ -352,21 +404,47 @@ sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
 		sec_bin_t *bin = &sec->bins[i];
 		malloc_mutex_lock(tsdn, &bin->mtx);
 		atomic_store_zu(&bin->bytes_cur, 0, ATOMIC_RELAXED);
+		atomic_store_zu(&bin->bytes_pinned_cur, 0, ATOMIC_RELAXED);
 		edata_list_active_concat(to_flush, &bin->freelist);
 		malloc_mutex_unlock(tsdn, &bin->mtx);
 	}
 }
 
+static void
+sec_bin_bytes_get(const sec_bin_t *bin, size_t *bytes, size_t *bytes_pinned) {
+	*bytes = atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+	*bytes_pinned = atomic_load_zu(
+	    &bin->bytes_pinned_cur, ATOMIC_RELAXED);
+	*bytes_pinned = min_zu(*bytes_pinned, *bytes);
+}
+
+static void
+sec_stats_merge_bin(
+    const sec_bin_t *bin, pszind_t pszind, sec_pszind_stats_t *stats) {
+	size_t bytes;
+	size_t bytes_pinned;
+	sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
+
+	stats->bytes += bytes;
+	stats->bytes_pinned += bytes_pinned;
+	size_t size = sz_pind2sz(pszind);
+	stats->nextents += bytes / size;
+	stats->nextents_pinned += bytes_pinned / size;
+}
+
 void
 sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
 	if (!sec_is_used(sec)) {
 		return;
 	}
-	size_t sum = 0;
 	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
 	for (pszind_t i = 0; i < ntotal_bins; i++) {
 		sec_bin_t *bin = &sec->bins[i];
-		sum += atomic_load_zu(&bin->bytes_cur, ATOMIC_RELAXED);
+		size_t bytes;
+		size_t bytes_pinned;
+		sec_bin_bytes_get(bin, &bytes, &bytes_pinned);
+		stats->bytes += bytes;
+		stats->bytes_pinned += bytes_pinned;
 		stats->total.nmisses +=
 		    atomic_load_zu(&bin->nmisses, ATOMIC_RELAXED);
 		stats->total.nhits +=
@@ -378,7 +456,20 @@ sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats) {
 		stats->total.noverfills +=
 		    atomic_load_zu(&bin->noverfills, ATOMIC_RELAXED);
 	}
-	stats->bytes += sum;
+}
+
+void
+sec_stats_merge_pszind(tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
+    sec_pszind_stats_t *stats) {
+	if (!sec_is_used(sec) || pszind >= sec->npsizes) {
+		return;
+	}
+
+	for (size_t shard = 0; shard < sec->opts.nshards; shard++) {
+		size_t ind = shard * sec->npsizes + pszind;
+		assert(ind < sec->npsizes * sec->opts.nshards);
+		sec_stats_merge_bin(&sec->bins[ind], pszind, stats);
+	}
 }
 
 void

src/stats.c

@@ -838,6 +838,37 @@ stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) {
 	    &sec_overfills);
 }
 
+static void
+stats_arena_pac_sec_print(emitter_t *emitter, unsigned i) {
+	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+	CTL_M2_GET("stats.arenas.0.pac_sec_bytes", i, &sec_bytes, size_t);
+	emitter_kv(emitter, "pac_sec_bytes",
+	    "Bytes in PAC small extent cache",
+	    emitter_type_size, &sec_bytes);
+	CTL_M2_GET("stats.arenas.0.pac_sec_hits", i, &sec_hits, size_t);
+	emitter_kv(emitter, "pac_sec_hits",
+	    "Total hits in PAC small extent cache",
+	    emitter_type_size, &sec_hits);
+	CTL_M2_GET("stats.arenas.0.pac_sec_misses", i, &sec_misses, size_t);
+	emitter_kv(emitter, "pac_sec_misses",
+	    "Total misses in PAC small extent cache",
+	    emitter_type_size, &sec_misses);
+	CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_noflush", i,
+	    &sec_dalloc_noflush, size_t);
+	emitter_kv(emitter, "pac_sec_dalloc_noflush",
+	    "Dalloc calls without flush in PAC small extent cache",
+	    emitter_type_size, &sec_dalloc_noflush);
+	CTL_M2_GET("stats.arenas.0.pac_sec_dalloc_flush", i, &sec_dalloc_flush,
+	    size_t);
+	emitter_kv(emitter, "pac_sec_dalloc_flush",
+	    "Dalloc calls with flush in PAC small extent cache",
+	    emitter_type_size, &sec_dalloc_flush);
+}
+
 static void
 stats_arena_hpa_shard_counters_print(
     emitter_t *emitter, unsigned i, uint64_t uptime) {
@@ -1570,6 +1601,10 @@ stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large,
 	GET_AND_EMIT_MEM_STAT(extent_avail)
 #undef GET_AND_EMIT_MEM_STAT
 
+	if (opt_pac_sec_opts.nshards > 0) {
+		stats_arena_pac_sec_print(emitter, i);
+	}
+
 	if (mutex) {
 		stats_arena_mutexes_print(emitter, i, uptime);
 	}
@@ -1764,6 +1799,9 @@ stats_general_print(emitter_t *emitter) {
 	OPT_WRITE_SIZE_T("hpa_sec_nshards")
 	OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
 	OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
+	OPT_WRITE_SIZE_T("experimental_pac_sec_nshards")
+	OPT_WRITE_SIZE_T("experimental_pac_sec_max_alloc")
+	OPT_WRITE_SIZE_T("experimental_pac_sec_max_bytes")
 	OPT_WRITE_BOOL("huge_arena_pac_thp")
 	OPT_WRITE_CHAR_P("metadata_thp")
 	OPT_WRITE_INT64("mutex_max_spin")

test/unit/json_stats.c

@@ -181,7 +181,7 @@ static const size_t num_global_mutexes = sizeof(global_mutex_names)
 static const char  *arena_mutex_names[] = {"large", "extent_avail",
      "extents_dirty", "extents_muzzy", "extents_retained", "decay_dirty",
      "decay_muzzy", "base", "tcache_list", "hpa_shard", "hpa_shard_grow",
-     "hpa_sec"};
+     "hpa_sec", "pac_sec"};
 static const size_t num_arena_mutexes = sizeof(arena_mutex_names)
     / sizeof(arena_mutex_names[0]);
 

test/unit/mallctl.c

@@ -530,6 +530,9 @@ TEST_BEGIN(test_mallctl_opt) {
 	TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always);
 	TEST_MALLCTL_OPT(size_t, hpa_sec_max_alloc, always);
 	TEST_MALLCTL_OPT(size_t, hpa_sec_max_bytes, always);
+	TEST_MALLCTL_OPT(size_t, experimental_pac_sec_nshards, always);
+	TEST_MALLCTL_OPT(size_t, experimental_pac_sec_max_alloc, always);
+	TEST_MALLCTL_OPT(size_t, experimental_pac_sec_max_bytes, always);
 	TEST_MALLCTL_OPT(ssize_t, experimental_hpa_max_purge_nhp, always);
 	TEST_MALLCTL_OPT(size_t, hpa_purge_threshold, always);
 	TEST_MALLCTL_OPT(uint64_t, hpa_min_purge_delay_ms, always);

test/unit/pac_sec_integration.c

@@ -0,0 +1,387 @@
+#include "test/jemalloc_test.h"
+
+/*
+ * Use multiple shards while keeping the single-threaded shard choice
+ * deterministic.  Disable background threads to avoid decay races.
+ */
+const char *malloc_conf =
+    "experimental_pac_sec_nshards:2,background_thread:false";
+
+static sec_opts_t saved_pac_sec_opts;
+
+static void
+pac_sec_test_opts_set(void) {
+	saved_pac_sec_opts = opt_pac_sec_opts;
+	/*
+	 * Include cache-oblivious padding so PAC can cache the test size.
+	 */
+	size_t test_extent_size = SC_LARGE_MINCLASS + sz_large_pad;
+	opt_pac_sec_opts.max_alloc = test_extent_size;
+	opt_pac_sec_opts.max_bytes = 4 * test_extent_size;
+}
+
+static void
+pac_sec_test_opts_restore(void) {
+	opt_pac_sec_opts = saved_pac_sec_opts;
+}
+
+static void *
+pinned_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr,
+    size_t size, size_t alignment, bool *zero, bool *commit,
+    unsigned arena_ind) {
+	void *ret = ehooks_default_extent_hooks.alloc(
+	    (extent_hooks_t *)&ehooks_default_extent_hooks, new_addr, size,
+	    alignment, zero, commit, arena_ind);
+	if (ret == NULL) {
+		return NULL;
+	}
+	if (!*commit) {
+		if (ehooks_default_extent_hooks.commit != NULL
+		    && ehooks_default_extent_hooks.commit(
+		        (extent_hooks_t *)&ehooks_default_extent_hooks, ret,
+		        size, 0, size, arena_ind)) {
+			ehooks_default_extent_hooks.dalloc(
+			    (extent_hooks_t *)&ehooks_default_extent_hooks, ret,
+			    size, *commit, arena_ind);
+			return NULL;
+		}
+		*commit = true;
+	}
+	return (void *)((uintptr_t)ret | EXTENT_ALLOC_FLAG_PINNED);
+}
+
+static void
+pinned_extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    bool committed, unsigned arena_ind) {
+	ehooks_default_extent_hooks.destroy(
+	    (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size,
+	    committed, arena_ind);
+}
+
+static bool
+pinned_extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
+    size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {
+	return ehooks_default_extent_hooks.split(
+	    (extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, size_a,
+	    size_b, committed, arena_ind);
+}
+
+static bool
+pinned_extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
+    void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
+	return ehooks_default_extent_hooks.merge(
+	    (extent_hooks_t *)&ehooks_default_extent_hooks, addr_a, size_a,
+	    addr_b, size_b, committed, arena_ind);
+}
+
+static extent_hooks_t pinned_hooks = {
+	pinned_extent_alloc,
+	NULL, /* dalloc */
+	pinned_extent_destroy,
+	NULL, /* commit */
+	NULL, /* decommit */
+	NULL, /* purge_lazy */
+	NULL, /* purge_forced */
+	pinned_extent_split,
+	pinned_extent_merge
+};
+
+static size_t
+read_arena_stat(unsigned arena_ind, const char *field) {
+	char   cmd[128];
+	size_t val;
+	size_t sz = sizeof(val);
+	uint64_t epoch = 1;
+	sz = sizeof(epoch);
+	expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0,
+	    "Unexpected mallctl failure");
+	sz = sizeof(val);
+	snprintf(cmd, sizeof(cmd), "stats.arenas.%u.%s", arena_ind, field);
+	expect_d_eq(mallctl(cmd, (void *)&val, &sz, NULL, 0), 0,
+	    "Unexpected mallctl failure reading arena stat");
+	return val;
+}
+
+static size_t
+read_stat(unsigned arena_ind, const char *field) {
+	char cmd[128];
+	snprintf(cmd, sizeof(cmd), "pac_sec_%s", field);
+	return read_arena_stat(arena_ind, cmd);
+}
+
+static size_t
+read_extents_stat(unsigned arena_ind, pszind_t pszind, const char *field) {
+	char cmd[128];
+	snprintf(cmd, sizeof(cmd), "extents.%u.%s", pszind, field);
+	return read_arena_stat(arena_ind, cmd);
+}
+
+static size_t
+read_pinned_npages(unsigned arena_ind) {
+	tsd_t *tsd = tsd_fetch();
+	arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
+	expect_ptr_not_null(arena, "arena_get failed");
+	return ecache_npages_get(&arena->pa_shard.pac.ecache_pinned);
+}
+
+static void
+dirty_decay_ms_set(unsigned arena_ind, ssize_t decay_ms) {
+	char cmd[64];
+	snprintf(cmd, sizeof(cmd), "arena.%u.dirty_decay_ms", arena_ind);
+	expect_d_eq(mallctl(cmd, NULL, NULL, (void *)&decay_ms,
+	    sizeof(decay_ms)), 0, "dirty_decay_ms mallctl failed");
+}
+
+TEST_BEGIN(test_pac_sec_alloc_dalloc_cycle) {
+	test_skip_if(!config_stats);
+	test_skip_if(opt_hpa);
+
+	pac_sec_test_opts_set();
+	unsigned arena_ind;
+	size_t   sz = sizeof(arena_ind);
+	expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Unexpected arenas.create failure");
+
+	int    flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
+	size_t alloc_size = SC_LARGE_MINCLASS;
+
+	size_t max_bytes;
+	sz = sizeof(max_bytes);
+	expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes",
+	    (void *)&max_bytes, &sz, NULL, 0), 0,
+	    "Unexpected mallctl failure");
+
+	size_t capacity = max_bytes / alloc_size;
+	expect_zu_gt(capacity, 0, "SEC capacity must be > 0 for this test");
+
+	/* Initial allocation misses SEC. */
+	void *p1 = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p1, "mallocx failed");
+	size_t resident_after_alloc = read_arena_stat(arena_ind, "resident");
+	expect_zu_eq(read_stat(arena_ind, "misses"), 1,
+	    "first alloc should miss SEC");
+	expect_zu_eq(read_stat(arena_ind, "hits"), 0,
+	    "no hits yet");
+	expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
+	    "SEC should be empty (extent is active)");
+
+	/* Freeing the extent caches it in PAC SEC. */
+	dallocx(p1, flags);
+	size_t cached_after_one = read_stat(arena_ind, "bytes");
+	expect_zu_gt(cached_after_one, 0,
+	    "SEC should cache the freed extent");
+	expect_zu_ge(read_arena_stat(arena_ind, "pdirty"),
+	    cached_after_one >> LG_PAGE,
+	    "SEC bytes should count toward dirty page stats");
+	expect_zu_ge(read_arena_stat(arena_ind, "resident"),
+	    resident_after_alloc,
+	    "SEC bytes should remain included in resident stats");
+	/* Use the actual extent size, including size class rounding. */
+	size_t extent_size = cached_after_one;
+	pszind_t extent_pszind = sz_psz2ind(extent_size);
+	expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "ndirty"), 1,
+	    "SEC extents should count toward extents dirty stats");
+	expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "dirty_bytes"),
+	    cached_after_one,
+	    "SEC bytes should count toward extents dirty byte stats");
+	expect_zu_eq(read_stat(arena_ind, "dalloc_noflush"), 1,
+	    "one dalloc absorbed without flush");
+	expect_zu_eq(read_stat(arena_ind, "dalloc_flush"), 0,
+	    "no flush yet");
+
+	/* Recompute capacity based on actual extent size. */
+	capacity = max_bytes / extent_size;
+	expect_zu_gt(capacity, 0, "SEC capacity should be positive");
+
+	/* Reallocate from SEC. */
+	void *p2 = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p2, "mallocx failed");
+	expect_zu_eq(read_stat(arena_ind, "hits"), 1,
+	    "second alloc should hit SEC");
+	expect_zu_eq(read_stat(arena_ind, "misses"), 1,
+	    "misses should not increase");
+	expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
+	    "SEC should be empty after hit");
+
+	dallocx(p2, flags);
+
+	/* Overflow SEC and flush cold extents to ecache_dirty. */
+	size_t nallocs = capacity + 2;
+	void **ptrs = mallocx(nallocs * sizeof(void *),
+	    MALLOCX_TCACHE_NONE);
+	expect_ptr_not_null(ptrs, "metadata alloc failed");
+
+	for (size_t i = 0; i < nallocs; i++) {
+		ptrs[i] = mallocx(alloc_size, flags);
+		expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i);
+	}
+	for (size_t i = 0; i < nallocs; i++) {
+		dallocx(ptrs[i], flags);
+	}
+
+	size_t noflush = read_stat(arena_ind, "dalloc_noflush");
+	size_t flush = read_stat(arena_ind, "dalloc_flush");
+	size_t cached_bytes = read_stat(arena_ind, "bytes");
+
+	expect_zu_gt(noflush, 1,
+	    "most dallocs should be absorbed");
+	expect_zu_gt(flush, 0,
+	    "overflow should trigger at least one flush");
+	expect_zu_gt(cached_bytes, 0,
+	    "SEC should still hold extents after partial flush");
+	expect_zu_le(cached_bytes, max_bytes,
+	    "SEC should not exceed max_bytes");
+
+	/* A populated SEC serves the next allocation. */
+	size_t misses_before = read_stat(arena_ind, "misses");
+	void *p3 = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p3, "mallocx failed");
+	expect_zu_eq(read_stat(arena_ind, "misses"), misses_before,
+	    "alloc from populated SEC should not miss");
+	dallocx(p3, flags);
+
+	/* Purge flushes SEC entirely. */
+	char cmd[64];
+	snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind);
+	expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
+	    "purge failed");
+	expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
+	    "SEC should be empty after purge");
+
+	/* Allocation after purge misses SEC. */
+	size_t hits_before = read_stat(arena_ind, "hits");
+	void *p4 = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p4, "mallocx failed");
+	expect_zu_eq(read_stat(arena_ind, "hits"), hits_before,
+	    "alloc after purge should miss SEC");
+	dallocx(p4, flags);
+
+	dallocx(ptrs, MALLOCX_TCACHE_NONE);
+	snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind);
+	expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
+	    "arena destroy failed");
+	pac_sec_test_opts_restore();
+}
+TEST_END
+
+TEST_BEGIN(test_pac_sec_dirty_decay_toggle) {
+	test_skip_if(!config_stats);
+	test_skip_if(opt_hpa);
+
+	pac_sec_test_opts_set();
+	unsigned arena_ind;
+	size_t sz = sizeof(arena_ind);
+	expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
+	    0, "Unexpected arenas.create failure");
+
+	int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
+	size_t alloc_size = SC_LARGE_MINCLASS;
+
+	void *p = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p, "mallocx failed");
+	dallocx(p, flags);
+	expect_zu_gt(read_stat(arena_ind, "bytes"), 0,
+	    "SEC should cache when dirty decay is enabled");
+
+	dirty_decay_ms_set(arena_ind, 0);
+	expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
+	    "disabling dirty decay should flush SEC");
+
+	p = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p, "mallocx failed");
+	dallocx(p, flags);
+	expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
+	    "SEC should stay disabled while dirty decay is zero");
+
+	dirty_decay_ms_set(arena_ind, 100);
+	p = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p, "mallocx failed");
+	dallocx(p, flags);
+	expect_zu_gt(read_stat(arena_ind, "bytes"), 0,
+	    "SEC should be usable after dirty decay is re-enabled");
+
+	char cmd[64];
+	snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind);
+	expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
+	    "arena destroy failed");
+	pac_sec_test_opts_restore();
+}
+TEST_END
+
+TEST_BEGIN(test_pac_sec_flush_pinned) {
+	test_skip_if(!config_stats);
+	test_skip_if(opt_hpa);
+
+	pac_sec_test_opts_set();
+	unsigned arena_ind;
+	size_t sz = sizeof(arena_ind);
+	extent_hooks_t *hooks_ptr = &pinned_hooks;
+	expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz,
+	    &hooks_ptr, sizeof(hooks_ptr)), 0,
+	    "Unexpected arenas.create failure");
+
+	int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
+	size_t alloc_size = SC_LARGE_MINCLASS;
+	size_t max_bytes;
+	sz = sizeof(max_bytes);
+	expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes",
+	    (void *)&max_bytes, &sz, NULL, 0), 0,
+	    "Unexpected mallctl failure");
+
+	void *p = mallocx(alloc_size, flags);
+	expect_ptr_not_null(p, "mallocx failed");
+	size_t resident_after_alloc = read_arena_stat(arena_ind, "resident");
+	dallocx(p, flags);
+	size_t sec_bytes = read_stat(arena_ind, "bytes");
+	expect_zu_gt(sec_bytes, 0, "SEC should cache the pinned extent");
+	expect_zu_ge(read_arena_stat(arena_ind, "pinned"), sec_bytes,
+	    "Pinned bytes cached in SEC should count toward pinned stats");
+	expect_zu_ge(read_arena_stat(arena_ind, "resident"),
+	    resident_after_alloc,
+	    "Pinned SEC bytes should remain included in resident stats");
+
+	size_t extent_size = sec_bytes;
+	pszind_t extent_pszind = sz_psz2ind(extent_size);
+	expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "npinned"), 1,
+	    "Pinned SEC extents should count toward extents pinned stats");
+	expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "pinned_bytes"),
+	    sec_bytes,
+	    "Pinned SEC bytes should count toward extents pinned byte stats");
+	size_t nallocs = max_bytes / extent_size + 2;
+	void **ptrs = mallocx(nallocs * sizeof(void *), MALLOCX_TCACHE_NONE);
+	expect_ptr_not_null(ptrs, "metadata alloc failed");
+	for (size_t i = 0; i < nallocs; i++) {
+		ptrs[i] = mallocx(alloc_size, flags);
+		expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i);
+	}
+	size_t pinned_before_overflow = read_pinned_npages(arena_ind);
+	for (size_t i = 0; i < nallocs; i++) {
+		dallocx(ptrs[i], flags);
+	}
+	expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_overflow,
+	    "SEC overflow should flush pinned extents to ecache_pinned");
+
+	size_t pinned_before_purge = read_pinned_npages(arena_ind);
+	char cmd[64];
+	snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind);
+	expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
+	    "purge failed");
+	expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
+	    "SEC should be empty after purge");
+	expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_purge,
+	    "PAC SEC purge should flush pinned extents to ecache_pinned");
+
+	dallocx(ptrs, MALLOCX_TCACHE_NONE);
+	snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind);
+	expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
+	    "arena destroy failed");
+	pac_sec_test_opts_restore();
+}
+TEST_END
+
+int
+main(void) {
+	return test_no_reentrancy(
+	    test_pac_sec_alloc_dalloc_cycle, test_pac_sec_dirty_decay_toggle,
+	    test_pac_sec_flush_pinned);
+}

test/unit/sec.c

@@ -12,6 +12,17 @@ struct test_data_s {
 	base_t *base;
 };
 
+static void
+edata_init_test(edata_t *edata) {
+	memset(edata, 0, sizeof(*edata));
+}
+
+static void
+edata_init_test_size(edata_t *edata, size_t size) {
+	edata_init_test(edata);
+	edata_size_set(edata, size);
+}
+
 static void
 test_data_init(tsdn_t *tsdn, test_data_t *tdata, const sec_opts_t *opts) {
 	tdata->base = base_new(TSDN_NULL, /* ind */ 123,
@@ -31,6 +42,32 @@ destroy_test_data(tsdn_t *tsdn, test_data_t *tdata) {
 	base_delete(tsdn, tdata->base);
 }
 
+static uint8_t
+test_sec_shard(tsdn_t *tsdn, sec_t *sec) {
+	if (tsdn_null(tsdn) || sec->opts.nshards <= 1) {
+		return 0;
+	}
+	tsd_t *tsd = tsdn_tsd(tsdn);
+	return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd));
+}
+
+static edata_t *
+sec_test_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
+	return sec_alloc(tsdn, sec, size, test_sec_shard(tsdn, sec));
+}
+
+static void
+sec_test_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
+    edata_list_active_t *result, size_t nallocs) {
+	sec_fill(tsdn, sec, size, result, nallocs, test_sec_shard(tsdn, sec));
+}
+
+static void
+sec_test_dalloc(tsdn_t *tsdn, sec_t *sec,
+    edata_list_active_t *dalloc_list) {
+	sec_dalloc(tsdn, sec, dalloc_list, test_sec_shard(tsdn, sec));
+}
+
 TEST_BEGIN(test_max_nshards_option_zero) {
 	test_data_t tdata;
 	sec_opts_t  opts;
@@ -41,7 +78,8 @@ TEST_BEGIN(test_max_nshards_option_zero) {
 	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
 	test_data_init(tsdn, &tdata, &opts);
 
-	edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE);
+	edata_t *edata = sec_size_supported(&tdata.sec, PAGE)
+	    ? sec_test_alloc(tsdn, &tdata.sec, PAGE) : NULL;
 	expect_ptr_null(edata, "SEC should be disabled when nshards==0");
 	destroy_test_data(tsdn, &tdata);
 }
@@ -57,7 +95,8 @@ TEST_BEGIN(test_max_alloc_option_too_small) {
 	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
 	test_data_init(tsdn, &tdata, &opts);
 
-	edata_t *edata = sec_alloc(tsdn, &tdata.sec, 3 * PAGE);
+	edata_t *edata = sec_size_supported(&tdata.sec, 3 * PAGE)
+	    ? sec_test_alloc(tsdn, &tdata.sec, 3 * PAGE) : NULL;
 	expect_ptr_null(edata, "max_alloc is 2*PAGE, should not alloc 3*PAGE");
 	destroy_test_data(tsdn, &tdata);
 }
@@ -78,11 +117,11 @@ TEST_BEGIN(test_sec_fill) {
 	edata_list_active_t allocs;
 	edata_list_active_init(&allocs);
 	edata_t edata1, edata2;
-	edata_size_set(&edata1, PAGE);
+	edata_init_test_size(&edata1, PAGE);
-	edata_size_set(&edata2, PAGE);
+	edata_init_test_size(&edata2, PAGE);
 	edata_list_active_append(&allocs, &edata1);
 	edata_list_active_append(&allocs, &edata2);
-	sec_fill(tsdn, &tdata.sec, PAGE, &allocs, 2);
+	sec_test_fill(tsdn, &tdata.sec, PAGE, &allocs, 2);
 	sec_stats_merge(tsdn, &tdata.sec, &stats);
 	expect_zu_eq(stats.bytes, 2 * PAGE, "SEC should have what we filled");
 	expect_true(edata_list_active_empty(&allocs),
@@ -91,13 +130,13 @@ TEST_BEGIN(test_sec_fill) {
 	/* Try to overfill and confirm that max_bytes is respected. */
 	stats.bytes = 0;
 	edata_t edata5, edata4, edata3;
-	edata_size_set(&edata3, PAGE);
+	edata_init_test_size(&edata3, PAGE);
-	edata_size_set(&edata4, PAGE);
+	edata_init_test_size(&edata4, PAGE);
-	edata_size_set(&edata5, PAGE);
+	edata_init_test_size(&edata5, PAGE);
 	edata_list_active_append(&allocs, &edata3);
 	edata_list_active_append(&allocs, &edata4);
 	edata_list_active_append(&allocs, &edata5);
-	sec_fill(tsdn, &tdata.sec, PAGE, &allocs, 3);
+	sec_test_fill(tsdn, &tdata.sec, PAGE, &allocs, 3);
 	sec_stats_merge(tsdn, &tdata.sec, &stats);
 	expect_zu_eq(
 	    stats.bytes, opts.max_bytes, "SEC can't have more than max_bytes");
@@ -118,20 +157,20 @@ TEST_BEGIN(test_sec_alloc) {
 	test_data_init(tsdn, &tdata, &opts);
 
 	/* Alloc from empty cache returns NULL */
-	edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE);
+	edata_t *edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
 	expect_ptr_null(edata, "SEC is empty");
 
 	/* Place two extents into the sec */
 	edata_list_active_t allocs;
 	edata_list_active_init(&allocs);
 	edata_t edata1, edata2;
-	edata_size_set(&edata1, PAGE);
+	edata_init_test_size(&edata1, PAGE);
 	edata_list_active_append(&allocs, &edata1);
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_true(edata_list_active_empty(&allocs), "");
-	edata_size_set(&edata2, PAGE);
+	edata_init_test_size(&edata2, PAGE);
 	edata_list_active_append(&allocs, &edata2);
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_true(edata_list_active_empty(&allocs), "");
 
 	sec_stats_t stats = {0};
@@ -141,20 +180,20 @@ TEST_BEGIN(test_sec_alloc) {
 	stats.bytes = 0;
 
 	/* Most recently cached extent should be used on alloc */
-	edata = sec_alloc(tsdn, &tdata.sec, PAGE);
+	edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
 	expect_ptr_eq(edata, &edata2, "edata2 is most recently used");
 	sec_stats_merge(tsdn, &tdata.sec, &stats);
 	expect_zu_eq(stats.bytes, PAGE, "One more item left in the cache");
 	stats.bytes = 0;
 
 	/* Alloc can still get extents from cache */
-	edata = sec_alloc(tsdn, &tdata.sec, PAGE);
+	edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
 	expect_ptr_eq(edata, &edata1, "SEC is not empty");
 	sec_stats_merge(tsdn, &tdata.sec, &stats);
 	expect_zu_eq(stats.bytes, 0, "No more items after last one is popped");
 
 	/* And cache is empty again */
-	edata = sec_alloc(tsdn, &tdata.sec, PAGE);
+	edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
 	expect_ptr_null(edata, "SEC is empty");
 	destroy_test_data(tsdn, &tdata);
 }
@@ -174,20 +213,20 @@ TEST_BEGIN(test_sec_dalloc) {
 	edata_list_active_t allocs;
 	edata_list_active_init(&allocs);
 	edata_t edata1;
-	edata_size_set(&edata1, PAGE);
+	edata_init_test_size(&edata1, PAGE);
 	edata_list_active_append(&allocs, &edata1);
 
 	/* SEC is empty, we return one pointer to it */
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_true(
 	    edata_list_active_empty(&allocs), "extents should be consumed");
 
 	/* Return one more extent, so that we are at the limit */
 	edata_t edata2;
-	edata_size_set(&edata2, PAGE);
+	edata_init_test_size(&edata2, PAGE);
 	edata_list_active_append(&allocs, &edata2);
 	/* Sec can take one more as well and we will be exactly at max_bytes */
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_true(
 	    edata_list_active_empty(&allocs), "extents should be consumed");
 
@@ -203,9 +242,9 @@ TEST_BEGIN(test_sec_dalloc) {
 	 * the one given in the input as it is the most recently used.
 	 */
 	edata_t edata3;
-	edata_size_set(&edata3, PAGE);
+	edata_init_test_size(&edata3, PAGE);
 	edata_list_active_append(&allocs, &edata3);
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_false(
 	    edata_list_active_empty(&allocs), "extents should NOT be consumed");
 	expect_ptr_ne(
@@ -218,6 +257,65 @@ TEST_BEGIN(test_sec_dalloc) {
 }
 TEST_END
 
+TEST_BEGIN(test_sec_pinned_stats) {
+	test_data_t tdata;
+	sec_opts_t  opts;
+	opts.nshards = 1;
+	opts.max_alloc = PAGE;
+	opts.max_bytes = 2 * PAGE;
+
+	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
+	test_data_init(tsdn, &tdata, &opts);
+
+	edata_list_active_t allocs;
+	edata_list_active_init(&allocs);
+	edata_t pinned, unpinned;
+	edata_init_test_size(&pinned, PAGE);
+	edata_pinned_set(&pinned, true);
+	edata_init_test_size(&unpinned, PAGE);
+
+	edata_list_active_append(&allocs, &pinned);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
+	edata_list_active_append(&allocs, &unpinned);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
+
+	sec_stats_t stats = {0};
+	sec_stats_merge(tsdn, &tdata.sec, &stats);
+	expect_zu_eq(stats.bytes, 2 * PAGE, "");
+	expect_zu_eq(stats.bytes_pinned, PAGE, "");
+	sec_pszind_stats_t pszind_stats = {0};
+	sec_stats_merge_pszind(
+	    tsdn, &tdata.sec, sz_psz2ind(PAGE), &pszind_stats);
+	expect_zu_eq(pszind_stats.nextents, 2, "");
+	expect_zu_eq(pszind_stats.bytes, 2 * PAGE, "");
+	expect_zu_eq(pszind_stats.nextents_pinned, 1, "");
+	expect_zu_eq(pszind_stats.bytes_pinned, PAGE, "");
+
+	memset(&stats, 0, sizeof(stats));
+	expect_ptr_eq(sec_test_alloc(tsdn, &tdata.sec, PAGE), &unpinned,
+	    "Unpinned extent is the most recently cached");
+	sec_stats_merge(tsdn, &tdata.sec, &stats);
+	expect_zu_eq(stats.bytes, PAGE, "");
+	expect_zu_eq(stats.bytes_pinned, PAGE, "");
+	memset(&pszind_stats, 0, sizeof(pszind_stats));
+	sec_stats_merge_pszind(
+	    tsdn, &tdata.sec, sz_psz2ind(PAGE), &pszind_stats);
+	expect_zu_eq(pszind_stats.nextents, 1, "");
+	expect_zu_eq(pszind_stats.bytes, PAGE, "");
+	expect_zu_eq(pszind_stats.nextents_pinned, 1, "");
+	expect_zu_eq(pszind_stats.bytes_pinned, PAGE, "");
+
+	memset(&stats, 0, sizeof(stats));
+	expect_ptr_eq(sec_test_alloc(tsdn, &tdata.sec, PAGE), &pinned,
+	    "Pinned extent should be tracked until allocated");
+	sec_stats_merge(tsdn, &tdata.sec, &stats);
+	expect_zu_eq(stats.bytes, 0, "");
+	expect_zu_eq(stats.bytes_pinned, 0, "");
+
+	destroy_test_data(tsdn, &tdata);
+}
+TEST_END
+
 TEST_BEGIN(test_max_bytes_too_low) {
 	test_data_t tdata;
 	sec_opts_t  opts;
@@ -232,11 +330,11 @@ TEST_BEGIN(test_max_bytes_too_low) {
 	edata_list_active_t allocs;
 	edata_list_active_init(&allocs);
 	edata_t edata1;
-	edata_size_set(&edata1, 3 * PAGE);
+	edata_init_test_size(&edata1, 3 * PAGE);
 	edata_list_active_append(&allocs, &edata1);
 
 	/* SEC is empty, we return one pointer to it */
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_false(
 	    edata_list_active_empty(&allocs), "extents should not be consumed");
 	destroy_test_data(tsdn, &tdata);
@@ -262,13 +360,13 @@ TEST_BEGIN(test_sec_flush) {
 	edata_t edata1[NALLOCS];
 	edata_t edata4[NALLOCS];
 	for (int i = 0; i < NALLOCS; i++) {
-		edata_size_set(&edata1[i], PAGE);
+		edata_init_test_size(&edata1[i], PAGE);
-		edata_size_set(&edata4[i], 4 * PAGE);
+		edata_init_test_size(&edata4[i], 4 * PAGE);
 
 		edata_list_active_append(&allocs1, &edata1[i]);
-		sec_dalloc(tsdn, &tdata.sec, &allocs1);
+		sec_test_dalloc(tsdn, &tdata.sec, &allocs1);
 		edata_list_active_append(&allocs4, &edata4[i]);
-		sec_dalloc(tsdn, &tdata.sec, &allocs4);
+		sec_test_dalloc(tsdn, &tdata.sec, &allocs4);
 	}
 
 	sec_stats_t stats = {0};
@@ -301,23 +399,23 @@ TEST_BEGIN(test_sec_stats) {
 	edata_list_active_t allocs;
 	edata_list_active_init(&allocs);
 	edata_t edata1;
-	edata_size_set(&edata1, PAGE);
+	edata_init_test_size(&edata1, PAGE);
 	edata_list_active_append(&allocs, &edata1);
 
 	/* SEC is empty alloc fails. nmisses==1 */
-	edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE);
+	edata_t *edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
 	expect_ptr_null(edata, "SEC should be empty");
 
 	/* SEC is empty, we return one pointer to it. ndalloc_noflush=1 */
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_true(
 	    edata_list_active_empty(&allocs), "extents should be consumed");
 
 	edata_t edata2;
-	edata_size_set(&edata2, PAGE);
+	edata_init_test_size(&edata2, PAGE);
 	edata_list_active_append(&allocs, &edata2);
 	/* Sec can take one more, so ndalloc_noflush=2 */
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_true(
 	    edata_list_active_empty(&allocs), "extents should be consumed");
 
@@ -335,9 +433,9 @@ TEST_BEGIN(test_sec_stats) {
 	 * the limit.  This will force flush, so ndalloc_flush = 1.
 	 */
 	edata_t edata3;
-	edata_size_set(&edata3, PAGE);
+	edata_init_test_size(&edata3, PAGE);
 	edata_list_active_append(&allocs, &edata3);
-	sec_dalloc(tsdn, &tdata.sec, &allocs);
+	sec_test_dalloc(tsdn, &tdata.sec, &allocs);
 	expect_false(
 	    edata_list_active_empty(&allocs), "extents should NOT be consumed");
 	sec_stats_merge(tsdn, &tdata.sec, &stats);
@@ -351,11 +449,6 @@ TEST_END
 #define NOPS_PER_THREAD 100
 #define NPREFILL 32
 
-static void
-edata_init_test(edata_t *edata) {
-	memset(edata, 0, sizeof(*edata));
-}
-
 typedef struct {
 	sec_t              *sec;
 	uint8_t             preferred_shard;
@@ -379,12 +472,12 @@ thd_trylock_test(void *varg) {
 	*shard_idx = arg->preferred_shard;
 
 	/* Fill the shard with some extents */
-	sec_fill(tsdn, arg->sec, PAGE, &arg->fill_list, arg->fill_list_sz);
+	sec_test_fill(tsdn, arg->sec, PAGE, &arg->fill_list, arg->fill_list_sz);
 	expect_true(edata_list_active_empty(&arg->fill_list), "");
 
 	for (unsigned i = 0; i < NOPS_PER_THREAD; i++) {
 		/* Try to allocate from SEC */
-		arg->edata[i] = sec_alloc(tsdn, arg->sec, PAGE);
+		arg->edata[i] = sec_test_alloc(tsdn, arg->sec, PAGE);
 		if (arg->edata[i] != NULL) {
 			expect_zu_eq(edata_size_get(arg->edata[i]), PAGE, "");
 		}
@@ -397,7 +490,7 @@ thd_trylock_test(void *varg) {
 			arg->nallocs++;
 			edata_list_active_append(&list, arg->edata[i]);
 			expect_zu_eq(edata_size_get(arg->edata[i]), PAGE, "");
-			sec_dalloc(tsdn, arg->sec, &list);
+			sec_test_dalloc(tsdn, arg->sec, &list);
 			if (edata_list_active_empty(&list)) {
 				arg->ndallocs++;
 			} else {
@@ -433,8 +526,7 @@ TEST_BEGIN(test_sec_multishard) {
 		edata_list_active_init(&args[i].fill_list);
 		for (unsigned j = 0; j < NPREFILL; ++j) {
 			size_t ind = i * NPREFILL + j;
-			edata_init_test(&all_edatas[ind]);
+			edata_init_test_size(&all_edatas[ind], PAGE);
-			edata_size_set(&all_edatas[ind], PAGE);
 			edata_list_active_append(
 			    &args[i].fill_list, &all_edatas[ind]);
 		}
@@ -487,6 +579,6 @@ int
 main(void) {
 	return test(test_max_nshards_option_zero,
 	    test_max_alloc_option_too_small, test_sec_fill, test_sec_alloc,
-	    test_sec_dalloc, test_max_bytes_too_low, test_sec_flush,
+	    test_sec_dalloc, test_sec_pinned_stats, test_max_bytes_too_low,
-	    test_sec_stats, test_sec_multishard);
+	    test_sec_flush, test_sec_stats, test_sec_multishard);
 }