Use SEC in PAC to reduce lock contention on the ecaches

Add a small extent cache in front of the PAC ecaches. Allocs and dallocs
that fit are served from per-shard SEC bins without taking the ecache
mutex; overflow falls through to the backing ecaches, including
ecache_pinned for pinned extents.

The feature is gated behind experimental_pac_sec_nshards (default 0,
disabled). To support independent HPA and PAC SEC instances,
sec_alloc/sec_dalloc/sec_fill take an explicit shard argument, with HPA
and PAC using separate TSD shard slots.

include/jemalloc/internal/jemalloc_internal_externs.h

@@ -21,6 +21,7 @@ extern bool             opt_confirm_conf;
 extern bool             opt_hpa;
 extern hpa_shard_opts_t opt_hpa_opts;
 extern sec_opts_t       opt_hpa_sec_opts;
+extern sec_opts_t       opt_pac_sec_opts;
 
 extern const char *opt_junk;
 extern bool        opt_junk_alloc;

include/jemalloc/internal/mutex_prof.h

@@ -37,7 +37,8 @@ typedef enum {
 	OP(tcache_list)                                                        \
 	OP(hpa_shard)                                                          \
 	OP(hpa_shard_grow)                                                     \
-	OP(hpa_sec)
+	OP(hpa_sec)                                                            \
+	OP(pac_sec)
 
 typedef enum {
 #define OP(mtx) arena_prof_mutex_##mtx,

include/jemalloc/internal/pac.h

@@ -8,6 +8,7 @@
 #include "jemalloc/internal/edata_cache.h"
 #include "jemalloc/internal/exp_grow.h"
 #include "jemalloc/internal/lockedint.h"
+#include "jemalloc/internal/sec.h"
 #include "jemalloc/internal/tsd_types.h"
 #include "san_bump.h"
 
@@ -84,12 +85,24 @@ struct pac_stats_s {
 
 	/* VM space had to be leaked (undocumented).  Normally 0. */
 	atomic_zu_t abandoned_vm;
+
+	/* PAC SEC stats.  Derived. */
+	sec_stats_t pac_sec_stats;
 };
 
 typedef struct pac_s pac_t;
 struct pac_s {
+	/* Small extent cache in front of PAC ecaches to reduce contention. */
+	sec_t sec;
+	/*
+	 * Runtime gate for PAC SEC.  0 disables (when SEC is not configured or
+	 * dirty_decay_ms == 0); otherwise mirrors sec.opts.max_alloc.
+	 */
+	atomic_zu_t sec_max_alloc;
+
 	/* True once pinned memory has been seen. */
 	atomic_b_t has_pinned;
+
 	/*
 	 * Collections of extents that were previously allocated.  These are
 	 * used when allocating extents, in an attempt to re-use address space.
@@ -236,4 +249,6 @@ ssize_t pac_decay_ms_get(pac_t *pac, extent_state_t state);
 
 void pac_destroy(tsdn_t *tsdn, pac_t *pac);
 
+void pac_sec_flush(tsdn_t *tsdn, pac_t *pac);
+
 #endif /* JEMALLOC_INTERNAL_PAC_H */

include/jemalloc/internal/sec.h

@@ -35,11 +35,21 @@ typedef struct sec_stats_s sec_stats_t;
 struct sec_stats_s {
 	/* Sum of bytes_cur across all shards. */
 	size_t bytes;
+	/* Subset of bytes that are pinned. */
+	size_t bytes_pinned;
 
 	/* Totals of bin_stats. */
 	sec_bin_stats_t total;
 };
 
+typedef struct sec_pszind_stats_s sec_pszind_stats_t;
+struct sec_pszind_stats_s {
+	size_t nextents;
+	size_t bytes;
+	size_t nextents_pinned;
+	size_t bytes_pinned;
+};
+
 static inline void
 sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
 	dst->nmisses += src->nmisses;
@@ -52,6 +62,7 @@ sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
 static inline void
 sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 	dst->bytes += src->bytes;
+	dst->bytes_pinned += src->bytes_pinned;
 	sec_bin_stats_accum(&dst->total, &src->total);
 }
 
@@ -67,6 +78,7 @@ struct sec_bin_s {
 	 * Number of bytes in this particular bin.
 	 */
 	atomic_zu_t         bytes_cur;
+	atomic_zu_t         bytes_pinned_cur;
 	edata_list_active_t freelist;
 	atomic_zu_t         nmisses;
 	atomic_zu_t         nhits;
@@ -88,8 +100,8 @@ sec_is_used(const sec_t *sec) {
 }
 
 static inline bool
-sec_size_supported(sec_t *sec, size_t size) {
-	return sec_is_used(sec) && size <= sec->opts.max_alloc;
+sec_size_supported(const sec_t *sec, size_t size) {
+	return size <= sec->opts.max_alloc;
 }
 
 /* Min number of extents we will allocate when expanding the SEC. */
@@ -111,12 +123,21 @@ sec_size_supported(sec_t *sec, size_t size) {
 void sec_calc_nallocs_for_size(
     sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
 
-/* If sec does not have extent available, it will return NULL. */
-edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
+/*
+ * Lazily picks (and caches in *idxp) a shard for the calling thread.  Different
+ * SEC instances pass independent per-thread uint8_t slots, initialized to
+ * (uint8_t)-1.
+ */
+uint8_t sec_shard_pick(tsd_t *tsd, sec_t *sec, uint8_t *idxp);
+
+/* Callers must ensure sec_size_supported(sec, size). */
+edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size, uint8_t shard);
 void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
-        edata_list_active_t *result, size_t nallocs);
+        edata_list_active_t *result, size_t nallocs, uint8_t shard);
 
 /*
+ * Callers must ensure sec_size_supported(sec, edata_size).
+ *
  * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty
  * if there are extents that need to be flushed from cache.  Please note, that
  * if we need to flush, extent(s) returned in the list to be deallocated
@@ -124,7 +145,8 @@ void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
  * considered "hot" and preserved in the cache, while "colder" ones are
  * returned).
  */
-void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list);
+void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list,
+    uint8_t shard);
 
 bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts);
 
@@ -138,6 +160,9 @@ void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
  * split), which simplifies the stats management.
  */
 void sec_stats_merge(tsdn_t *tsdn, const sec_t *sec, sec_stats_t *stats);
+void sec_stats_merge_pszind(
+    tsdn_t *tsdn, const sec_t *sec, pszind_t pszind,
+    sec_pszind_stats_t *stats);
 void sec_mutex_stats_read(
     tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data);
 

include/jemalloc/internal/tsd_internals.h

@@ -83,6 +83,7 @@ typedef void (*test_callback_t)(int *);
 	O(arena, arena_t *, arena_t *)                                         \
 	O(arena_decay_ticker, ticker_geom_t, ticker_geom_t)                    \
 	O(sec_shard, uint8_t, uint8_t)                                         \
+	O(pac_sec_shard, uint8_t, uint8_t)                                     \
 	O(binshards, tsd_binshards_t, tsd_binshards_t)                         \
 	O(peak, peak_t, peak_t)                                                \
 	O(tcache_slow, tcache_slow_t, tcache_slow_t)                           \
@@ -102,6 +103,7 @@ typedef void (*test_callback_t)(int *);
 	    /* arena */ NULL, /* arena_decay_ticker */                         \
 	    TICKER_GEOM_INIT(ARENA_DECAY_NTICKS_PER_UPDATE),                   \
 	    /* sec_shard */ (uint8_t) - 1,                                     \
+	    /* pac_sec_shard */ (uint8_t) - 1,                                 \
 	    /* binshards */ TSD_BINSHARDS_ZERO_INITIALIZER,                    \
 	    /* peak */ PEAK_INITIALIZER, /* tcache_slow */                     \
 	    TCACHE_SLOW_ZERO_INITIALIZER,                                      \