[SEC] Make SEC owned by hpa_shard, simplify the code, add stats, lock per bin

Makefile.in

@@ -135,7 +135,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/nstime.c \
 	$(srcroot)src/pa.c \
 	$(srcroot)src/pa_extra.c \
-	$(srcroot)src/pai.c \
 	$(srcroot)src/pac.c \
 	$(srcroot)src/pages.c \
 	$(srcroot)src/peak_event.c \
@@ -230,6 +229,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/hash.c \
 	$(srcroot)test/unit/hook.c \
 	$(srcroot)test/unit/hpa.c \
+	$(srcroot)test/unit/hpa_sec_integration.c \
 	$(srcroot)test/unit/hpa_thp_always.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \

include/jemalloc/internal/arena_externs.h

@@ -46,7 +46,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
-    hpa_shard_stats_t *hpastats, sec_stats_t *secstats);
+    hpa_shard_stats_t *hpastats);
 void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
 edata_t *arena_extent_alloc_large(
     tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);

include/jemalloc/internal/ctl.h

@@ -51,7 +51,6 @@ typedef struct ctl_arena_stats_s {
 	arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
 	pac_estats_t        estats[SC_NPSIZES];
 	hpa_shard_stats_t   hpastats;
-	sec_stats_t         secstats;
 } ctl_arena_stats_t;
 
 typedef struct ctl_stats_s {

include/jemalloc/internal/hpa.h

@@ -12,6 +12,7 @@
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/pai.h"
 #include "jemalloc/internal/psset.h"
+#include "jemalloc/internal/sec.h"
 
 typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
 struct hpa_shard_nonderived_stats_s {
@@ -57,6 +58,7 @@ typedef struct hpa_shard_stats_s hpa_shard_stats_t;
 struct hpa_shard_stats_s {
 	psset_stats_t                psset_stats;
 	hpa_shard_nonderived_stats_t nonderived_stats;
+	sec_stats_t                  secstats;
 };
 
 typedef struct hpa_shard_s hpa_shard_t;
@@ -69,14 +71,17 @@ struct hpa_shard_s {
 
 	/* The central allocator we get our hugepages from. */
 	hpa_central_t *central;
+
 	/* Protects most of this shard's state. */
 	malloc_mutex_t mtx;
+
 	/*
 	 * Guards the shard's access to the central allocator (preventing
 	 * multiple threads operating on this shard from accessing the central
 	 * allocator).
 	 */
 	malloc_mutex_t grow_mtx;
+
 	/* The base metadata allocator. */
 	base_t *base;
 
@@ -87,6 +92,9 @@ struct hpa_shard_s {
 	 */
 	edata_cache_fast_t ecf;
 
+	/* Small extent cache (not guarded by mtx) */
+	JEMALLOC_ALIGNED(CACHELINE) sec_t sec;
+
 	psset_t psset;
 
 	/*
@@ -142,9 +150,9 @@ bool hpa_hugepage_size_exceeds_limit(void);
  * just that it can function properly given the system it's running on.
  */
 bool hpa_supported(void);
-bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
+bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
-    base_t *base, edata_cache_t *edata_cache, unsigned ind,
+    emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
-    const hpa_shard_opts_t *opts);
+    const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts);
 
 void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
 void hpa_shard_stats_merge(
@@ -157,6 +165,8 @@ void hpa_shard_stats_merge(
  */
 void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard);
+/* Flush caches that shard may be using */
+void hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard);
 
 void hpa_shard_set_deferral_allowed(
     tsdn_t *tsdn, hpa_shard_t *shard, bool deferral_allowed);
@@ -164,8 +174,9 @@ void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
 
 /*
  * We share the fork ordering with the PA and arena prefork handling; that's why
- * these are 3 and 4 rather than 0 and 1.
+ * these are 2, 3 and 4 rather than 0 and 1.
  */
+void hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard);
 void hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard);

include/jemalloc/internal/pa.h

@@ -96,12 +96,6 @@ struct pa_shard_s {
 	/* Allocates from a PAC. */
 	pac_t pac;
 
-	/*
-	 * We place a small extent cache in front of the HPA, since we intend
-	 * these configurations to use many fewer arenas, and therefore have a
-	 * higher risk of hot locks.
-	 */
-	sec_t       hpa_sec;
 	hpa_shard_t hpa_shard;
 
 	/* The source of edata_t objects. */
@@ -166,6 +160,9 @@ void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard);
  */
 void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
 
+/* Flush any caches used by shard */
+void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard);
+
 /* Gets an edata for the given allocation. */
 edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
     size_t alignment, bool slab, szind_t szind, bool zero, bool guarded,
@@ -233,8 +230,7 @@ void pa_shard_basic_stats_merge(
 
 void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
-    hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
+    hpa_shard_stats_t *hpa_stats_out, size_t *resident);
-    size_t *resident);
 
 /*
  * Reads the PA-owned mutex stats into the output stats array, at the

include/jemalloc/internal/pai.h

@@ -13,15 +13,6 @@ struct pai_s {
 	edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
 	    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
 	    bool *deferred_work_generated);
-	/*
-	 * Returns the number of extents added to the list (which may be fewer
-	 * than requested, in case of OOM).  The list should already be
-	 * initialized.  The only alignment guarantee is page-alignment, and
-	 * the results are not necessarily zeroed.
-	 */
-	size_t (*alloc_batch)(tsdn_t *tsdn, pai_t *self, size_t size,
-	    size_t nallocs, edata_list_active_t *results, bool frequent_reuse,
-	    bool *deferred_work_generated);
 	bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
 	    size_t old_size, size_t new_size, bool zero,
 	    bool *deferred_work_generated);
@@ -29,9 +20,6 @@ struct pai_s {
 	    size_t old_size, size_t new_size, bool *deferred_work_generated);
 	void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
 	    bool *deferred_work_generated);
-	/* This function empties out list as a side-effect of being called. */
-	void (*dalloc_batch)(tsdn_t *tsdn, pai_t *self,
-	    edata_list_active_t *list, bool *deferred_work_generated);
 	uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self);
 };
 
@@ -47,14 +35,6 @@ pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
 	    frequent_reuse, deferred_work_generated);
 }
 
-static inline size_t
-pai_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
-    edata_list_active_t *results, bool frequent_reuse,
-    bool *deferred_work_generated) {
-	return self->alloc_batch(tsdn, self, size, nallocs, results,
-	    frequent_reuse, deferred_work_generated);
-}
-
 static inline bool
 pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
     size_t new_size, bool zero, bool *deferred_work_generated) {
@@ -75,26 +55,9 @@ pai_dalloc(
 	self->dalloc(tsdn, self, edata, deferred_work_generated);
 }
 
-static inline void
-pai_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
-    bool *deferred_work_generated) {
-	self->dalloc_batch(tsdn, self, list, deferred_work_generated);
-}
-
 static inline uint64_t
 pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
 	return self->time_until_deferred_work(tsdn, self);
 }
 
-/*
- * An implementation of batch allocation that simply calls alloc once for
- * each item in the list.
- */
-size_t pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size,
-    size_t nallocs, edata_list_active_t *results, bool frequent_reuse,
-    bool *deferred_work_generated);
-/* Ditto, for dalloc. */
-void pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self,
-    edata_list_active_t *list, bool *deferred_work_generated);
-
 #endif /* JEMALLOC_INTERNAL_PAI_H */

include/jemalloc/internal/sec.h

@@ -17,91 +17,104 @@
  * knowledge of the underlying PAI implementation).
  */
 
-/*
+typedef struct sec_bin_stats_s sec_bin_stats_t;
- * For now, this is just one field; eventually, we'll probably want to get more
+struct sec_bin_stats_s {
- * fine-grained data out (like per-size class statistics).
+	/* Number of alloc requests that did not find extent in this bin */
- */
+	size_t nmisses;
+	/* Number of successful alloc requests. */
+	size_t nhits;
+	/* Number of dallocs causing the flush */
+	size_t ndalloc_flush;
+	/* Number of dallocs not causing the flush */
+	size_t ndalloc_noflush;
+	/* Number of fills that hit max_bytes */
+	size_t noverfills;
+};
 typedef struct sec_stats_s sec_stats_t;
 struct sec_stats_s {
 	/* Sum of bytes_cur across all shards. */
 	size_t bytes;
+
+	/* Totals of bin_stats. */
+	sec_bin_stats_t total;
 };
 
+static inline void
+sec_bin_stats_init(sec_bin_stats_t *stats) {
+	stats->ndalloc_flush = 0;
+	stats->nmisses = 0;
+	stats->nhits = 0;
+	stats->ndalloc_noflush = 0;
+	stats->noverfills = 0;
+}
+
+static inline void
+sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
+	dst->nmisses += src->nmisses;
+	dst->nhits += src->nhits;
+	dst->ndalloc_flush += src->ndalloc_flush;
+	dst->ndalloc_noflush += src->ndalloc_noflush;
+	dst->noverfills += src->noverfills;
+}
+
 static inline void
 sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
 	dst->bytes += src->bytes;
+	sec_bin_stats_accum(&dst->total, &src->total);
 }
 
 /* A collections of free extents, all of the same size. */
 typedef struct sec_bin_s sec_bin_t;
 struct sec_bin_s {
 	/*
-	 * When we fail to fulfill an allocation, we do a batch-alloc on the
+	 * Protects the data members of the bin.
-	 * underlying allocator to fill extra items, as well.  We drop the SEC
-	 * lock while doing so, to allow operations on other bins to succeed.
-	 * That introduces the possibility of other threads also trying to
-	 * allocate out of this bin, failing, and also going to the backing
-	 * allocator.  To avoid a thundering herd problem in which lots of
-	 * threads do batch allocs and overfill this bin as a result, we only
-	 * allow one batch allocation at a time for a bin.  This bool tracks
-	 * whether or not some thread is already batch allocating.
-	 *
-	 * Eventually, the right answer may be a smarter sharding policy for the
-	 * bins (e.g. a mutex per bin, which would also be more scalable
-	 * generally; the batch-allocating thread could hold it while
-	 * batch-allocating).
 	 */
-	bool being_batch_filled;
+	malloc_mutex_t mtx;
 
 	/*
-	 * Number of bytes in this particular bin (as opposed to the
+	 * Number of bytes in this particular bin.
-	 * sec_shard_t's bytes_cur.  This isn't user visible or reported in
-	 * stats; rather, it allows us to quickly determine the change in the
-	 * centralized counter when flushing.
 	 */
 	size_t              bytes_cur;
 	edata_list_active_t freelist;
-};
+	sec_bin_stats_t     stats;
-
-typedef struct sec_shard_s sec_shard_t;
-struct sec_shard_s {
-	/*
-	 * We don't keep per-bin mutexes, even though that would allow more
-	 * sharding; this allows global cache-eviction, which in turn allows for
-	 * better balancing across free lists.
-	 */
-	malloc_mutex_t mtx;
-	/*
-	 * A SEC may need to be shut down (i.e. flushed of its contents and
-	 * prevented from further caching).  To avoid tricky synchronization
-	 * issues, we just track enabled-status in each shard, guarded by a
-	 * mutex.  In practice, this is only ever checked during brief races,
-	 * since the arena-level atomic boolean tracking HPA enabled-ness means
-	 * that we won't go down these pathways very often after custom extent
-	 * hooks are installed.
-	 */
-	bool       enabled;
-	sec_bin_t *bins;
-	/* Number of bytes in all bins in the shard. */
-	size_t bytes_cur;
-	/* The next pszind to flush in the flush-some pathways. */
-	pszind_t to_flush_next;
 };
 
 typedef struct sec_s sec_t;
 struct sec_s {
-	pai_t  pai;
+	sec_opts_t opts;
-	pai_t *fallback;
+	sec_bin_t *bins;
-
+	pszind_t   npsizes;
-	sec_opts_t   opts;
-	sec_shard_t *shards;
-	pszind_t     npsizes;
 };
 
-bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
+static inline bool
-    const sec_opts_t *opts);
+sec_is_used(sec_t *sec) {
-void sec_flush(tsdn_t *tsdn, sec_t *sec);
+	return sec->opts.nshards != 0;
-void sec_disable(tsdn_t *tsdn, sec_t *sec);
+}
+
+static inline bool
+sec_size_supported(sec_t *sec, size_t size) {
+	return sec_is_used(sec) && size <= sec->opts.max_alloc;
+}
+
+/* If sec does not have extent available, it will return NULL. */
+edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
+void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
+        edata_list_active_t *result, size_t nallocs);
+
+/*
+ * Upon return dalloc_list may be empty if edata is consumed by sec or non-empty
+ * if there are extents that need to be flushed from cache.  Please note, that
+ * if we need to flush, extent(s) returned in the list to be deallocated
+ * will almost certainly not contain the one being dalloc-ed (that one will be
+ * considered "hot" and preserved in the cache, while "colder" ones are
+ * returned).
+ */
+void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list);
+
+bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts);
+
+/* Fills to_flush with extents that need to be deallocated */
+void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
 
 /*
  * Morally, these two stats methods probably ought to be a single one (and the

include/jemalloc/internal/sec_opts.h

@@ -12,46 +12,39 @@ typedef struct sec_opts_s sec_opts_t;
 struct sec_opts_s {
 	/*
 	 * We don't necessarily always use all the shards; requests are
-	 * distributed across shards [0, nshards - 1).
+	 * distributed across shards [0, nshards - 1).  Once thread picks a
+	 * shard it will always use that one.  If this value is set to 0 sec is
+	 * not used.
 	 */
 	size_t nshards;
 	/*
 	 * We'll automatically refuse to cache any objects in this sec if
-	 * they're larger than max_alloc bytes, instead forwarding such objects
+	 * they're larger than max_alloc bytes.
-	 * directly to the fallback.
 	 */
 	size_t max_alloc;
 	/*
-	 * Exceeding this amount of cached extents in a shard causes us to start
+	 * Exceeding this amount of cached extents in a bin causes us to flush
-	 * flushing bins in that shard until we fall below bytes_after_flush.
+	 * until we are 1/4 below max_bytes.
 	 */
 	size_t max_bytes;
-	/*
-	 * The number of bytes (in all bins) we flush down to when we exceed
-	 * bytes_cur.  We want this to be less than bytes_cur, because
-	 * otherwise we could get into situations where a shard undergoing
-	 * net-deallocation keeps bytes_cur very near to max_bytes, so that
-	 * most deallocations get immediately forwarded to the underlying PAI
-	 * implementation, defeating the point of the SEC.
-	 */
-	size_t bytes_after_flush;
 	/*
 	 * When we can't satisfy an allocation out of the SEC because there are
-	 * no available ones cached, we allocate multiple of that size out of
+	 * no available ones cached, allocator will allocate a batch with extra
-	 * the fallback allocator.  Eventually we might want to do something
+	 * batch_fill_extra extents of the same size.
-	 * cleverer, but for now we just grab a fixed number.
 	 */
 	size_t batch_fill_extra;
 };
 
+#define SEC_OPTS_NSHARDS_DEFAULT 2
+#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3
+#define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024))
+#define SEC_OPTS_MAX_BYTES_DEFAULT                                             \
+	((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT)                       \
+	        ? (4 * SEC_OPTS_MAX_ALLOC_DEFAULT)                             \
+	        : (256 * 1024))
+
 #define SEC_OPTS_DEFAULT                                                       \
-	{                                                                      \
+	{SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT,                 \
-		/* nshards */                                                  \
+	    SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT}
-		4,                                           /* max_alloc */   \
-		    (32 * 1024) < PAGE ? PAGE : (32 * 1024), /* max_bytes */   \
-		    256 * 1024, /* bytes_after_flush */                        \
-		    128 * 1024, /* batch_fill_extra */                         \
-		    0                                                          \
-	}
 
 #endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

include/jemalloc/internal/witness.h

@@ -46,7 +46,7 @@ enum witness_rank_e {
 	WITNESS_RANK_DECAY = WITNESS_RANK_CORE,
 	WITNESS_RANK_TCACHE_QL,
 
-	WITNESS_RANK_SEC_SHARD,
+	WITNESS_RANK_SEC_BIN,
 
 	WITNESS_RANK_EXTENT_GROW,
 	WITNESS_RANK_HPA_SHARD_GROW = WITNESS_RANK_EXTENT_GROW,

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -74,7 +74,6 @@
     <ClCompile Include="..\..\..\..\src\nstime.c" />
     <ClCompile Include="..\..\..\..\src\pa.c" />
     <ClCompile Include="..\..\..\..\src\pa_extra.c" />
-    <ClCompile Include="..\..\..\..\src\pai.c" />
     <ClCompile Include="..\..\..\..\src\pac.c" />
     <ClCompile Include="..\..\..\..\src\pages.c" />
     <ClCompile Include="..\..\..\..\src\peak_event.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -106,9 +106,6 @@
     <ClCompile Include="..\..\..\..\src\pa_extra.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\pai.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\pac.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -74,7 +74,6 @@
     <ClCompile Include="..\..\..\..\src\nstime.c" />
     <ClCompile Include="..\..\..\..\src\pa.c" />
     <ClCompile Include="..\..\..\..\src\pa_extra.c" />
-    <ClCompile Include="..\..\..\..\src\pai.c" />
     <ClCompile Include="..\..\..\..\src\pac.c" />
     <ClCompile Include="..\..\..\..\src\pages.c" />
     <ClCompile Include="..\..\..\..\src\peak_event.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -106,9 +106,6 @@
     <ClCompile Include="..\..\..\..\src\pa_extra.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\pai.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\pac.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj

@@ -74,7 +74,6 @@
     <ClCompile Include="..\..\..\..\src\nstime.c" />
     <ClCompile Include="..\..\..\..\src\pa.c" />
     <ClCompile Include="..\..\..\..\src\pa_extra.c" />
-    <ClCompile Include="..\..\..\..\src\pai.c" />
     <ClCompile Include="..\..\..\..\src\pac.c" />
     <ClCompile Include="..\..\..\..\src\pages.c" />
     <ClCompile Include="..\..\..\..\src\peak_event.c" />

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters

@@ -106,9 +106,6 @@
     <ClCompile Include="..\..\..\..\src\pa_extra.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\pai.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\pac.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj

@@ -74,7 +74,6 @@
     <ClCompile Include="..\..\..\..\src\nstime.c" />
     <ClCompile Include="..\..\..\..\src\pa.c" />
     <ClCompile Include="..\..\..\..\src\pa_extra.c" />
-    <ClCompile Include="..\..\..\..\src\pai.c" />
     <ClCompile Include="..\..\..\..\src\pac.c" />
     <ClCompile Include="..\..\..\..\src\pages.c" />
     <ClCompile Include="..\..\..\..\src\peak_event.c" />

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters

@@ -106,9 +106,6 @@
     <ClCompile Include="..\..\..\..\src\pa_extra.c">
       <Filter>Source Files</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\..\..\src\pai.c">
-      <Filter>Source Files</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\..\..\src\pac.c">
       <Filter>Source Files</Filter>
     </ClCompile>

src/arena.c

@@ -89,7 +89,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
     const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
     size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
     bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
-    hpa_shard_stats_t *hpastats, sec_stats_t *secstats) {
+    hpa_shard_stats_t *hpastats) {
 	cassert(config_stats);
 
 	arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms,
@@ -159,7 +159,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
 	}
 
 	pa_shard_stats_merge(tsdn, &arena->pa_shard, &astats->pa_shard_stats,
-	    estats, hpastats, secstats, &astats->resident);
+	    estats, hpastats, &astats->resident);
 
 	LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
 
@@ -529,7 +529,7 @@ arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
 		 * as possible", including flushing any caches (for situations
 		 * like thread death, or manual purge calls).
 		 */
-		sec_flush(tsdn, &arena->pa_shard.hpa_sec);
+		pa_shard_flush(tsdn, &arena->pa_shard);
 	}
 	if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
 		return;

src/ctl.c

@@ -115,7 +115,6 @@ CTL_PROTO(opt_hpa_dirty_mult)
 CTL_PROTO(opt_hpa_sec_nshards)
 CTL_PROTO(opt_hpa_sec_max_alloc)
 CTL_PROTO(opt_hpa_sec_max_bytes)
-CTL_PROTO(opt_hpa_sec_bytes_after_flush)
 CTL_PROTO(opt_hpa_sec_batch_fill_extra)
 CTL_PROTO(opt_huge_arena_pac_thp)
 CTL_PROTO(opt_metadata_thp)
@@ -339,6 +338,11 @@ CTL_PROTO(stats_arenas_i_tcache_stashed_bytes)
 CTL_PROTO(stats_arenas_i_resident)
 CTL_PROTO(stats_arenas_i_abandoned_vm)
 CTL_PROTO(stats_arenas_i_hpa_sec_bytes)
+CTL_PROTO(stats_arenas_i_hpa_sec_hits)
+CTL_PROTO(stats_arenas_i_hpa_sec_misses)
+CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_flush)
+CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_noflush)
+CTL_PROTO(stats_arenas_i_hpa_sec_overfills)
 INDEX_PROTO(stats_arenas_i)
 CTL_PROTO(stats_allocated)
 CTL_PROTO(stats_active)
@@ -486,7 +490,6 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
     {NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
     {NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
     {NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
-    {NAME("hpa_sec_bytes_after_flush"), CTL(opt_hpa_sec_bytes_after_flush)},
     {NAME("hpa_sec_batch_fill_extra"), CTL(opt_hpa_sec_batch_fill_extra)},
     {NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
     {NAME("metadata_thp"), CTL(opt_metadata_thp)},
@@ -826,6 +829,12 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
     {NAME("resident"), CTL(stats_arenas_i_resident)},
     {NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)},
     {NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)},
+    {NAME("hpa_sec_hits"), CTL(stats_arenas_i_hpa_sec_hits)},
+    {NAME("hpa_sec_misses"), CTL(stats_arenas_i_hpa_sec_misses)},
+    {NAME("hpa_sec_dalloc_noflush"),
+        CTL(stats_arenas_i_hpa_sec_dalloc_noflush)},
+    {NAME("hpa_sec_dalloc_flush"), CTL(stats_arenas_i_hpa_sec_dalloc_flush)},
+    {NAME("hpa_sec_overfills"), CTL(stats_arenas_i_hpa_sec_overfills)},
     {NAME("small"), CHILD(named, stats_arenas_i_small)},
     {NAME("large"), CHILD(named, stats_arenas_i_large)},
     {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
@@ -1066,7 +1075,7 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
 		    &ctl_arena->pdirty, &ctl_arena->pmuzzy,
 		    &ctl_arena->astats->astats, ctl_arena->astats->bstats,
 		    ctl_arena->astats->lstats, ctl_arena->astats->estats,
-		    &ctl_arena->astats->hpastats, &ctl_arena->astats->secstats);
+		    &ctl_arena->astats->hpastats);
 
 		for (i = 0; i < SC_NBINS; i++) {
 			bin_stats_t *bstats =
@@ -1258,7 +1267,6 @@ ctl_arena_stats_sdmerge(
 
 		/* Merge HPA stats. */
 		hpa_shard_stats_accum(&sdstats->hpastats, &astats->hpastats);
-		sec_stats_accum(&sdstats->secstats, &astats->secstats);
 	}
 }
 
@@ -2175,11 +2183,8 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
 CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
-CTL_RO_NL_GEN(
-    opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush, size_t)
 CTL_RO_NL_GEN(
     opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, size_t)
-
 CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
 CTL_RO_NL_GEN(
     opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
@@ -3869,7 +3874,17 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm,
     size_t)
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_bytes,
-    arenas_i(mib[2])->astats->secstats.bytes, size_t)
+    arenas_i(mib[2])->astats->hpastats.secstats.bytes, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_hits,
+    arenas_i(mib[2])->astats->hpastats.secstats.total.nhits, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_misses,
+    arenas_i(mib[2])->astats->hpastats.secstats.total.nmisses, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_flush,
+    arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_flush, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_noflush,
+    arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_noflush, size_t)
+CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_overfills,
+    arenas_i(mib[2])->astats->hpastats.secstats.total.noverfills, size_t)
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
     arenas_i(mib[2])->astats->allocated_small, size_t)

src/hpa.c

@@ -11,19 +11,17 @@
 static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
     size_t alignment, bool zero, bool guarded, bool frequent_reuse,
     bool *deferred_work_generated);
-static size_t   hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size,
-      size_t nallocs, edata_list_active_t *results, bool frequent_reuse,
-      bool *deferred_work_generated);
 static bool     hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
         size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
 static bool     hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
         size_t old_size, size_t new_size, bool *deferred_work_generated);
 static void     hpa_dalloc(
         tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
-static void     hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
-        edata_list_active_t *list, bool *deferred_work_generated);
 static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
 
+static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
+    edata_list_active_t *list, bool *deferred_work_generated);
+
 const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
 
 bool opt_experimental_hpa_start_huge_if_thp_always = true;
@@ -74,9 +72,9 @@ hpa_do_consistency_checks(hpa_shard_t *shard) {
 }
 
 bool
-hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
+hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
-    base_t *base, edata_cache_t *edata_cache, unsigned ind,
+    emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
-    const hpa_shard_opts_t *opts) {
+    const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts) {
 	/* malloc_conf processing should have filtered out these cases. */
 	assert(hpa_supported());
 	bool err;
@@ -118,13 +116,16 @@ hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
 	 * operating on corrupted data.
 	 */
 	shard->pai.alloc = &hpa_alloc;
-	shard->pai.alloc_batch = &hpa_alloc_batch;
 	shard->pai.expand = &hpa_expand;
 	shard->pai.shrink = &hpa_shrink;
 	shard->pai.dalloc = &hpa_dalloc;
-	shard->pai.dalloc_batch = &hpa_dalloc_batch;
 	shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work;
 
+	err = sec_init(tsdn, &shard->sec, base, sec_opts);
+	if (err) {
+		return true;
+	}
+
 	hpa_do_consistency_checks(shard);
 
 	return false;
@@ -151,6 +152,7 @@ hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src) {
 	psset_stats_accum(&dst->psset_stats, &src->psset_stats);
 	hpa_shard_nonderived_stats_accum(
 	    &dst->nonderived_stats, &src->nonderived_stats);
+	sec_stats_accum(&dst->secstats, &src->secstats);
 }
 
 void
@@ -164,6 +166,8 @@ hpa_shard_stats_merge(
 	hpa_shard_nonderived_stats_accum(&dst->nonderived_stats, &shard->stats);
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 	malloc_mutex_unlock(tsdn, &shard->grow_mtx);
+
+	sec_stats_merge(tsdn, &shard->sec, &dst->secstats);
 }
 
 static bool
@@ -825,37 +829,9 @@ hpa_from_pai(pai_t *self) {
 	return (hpa_shard_t *)self;
 }
 
-static size_t
+static void
-hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
+hpa_assert_results(
-    edata_list_active_t *results, bool frequent_reuse,
+    tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *results) {
-    bool *deferred_work_generated) {
-	assert(nallocs > 0);
-	assert((size & PAGE_MASK) == 0);
-	witness_assert_depth_to_rank(
-	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-	hpa_shard_t *shard = hpa_from_pai(self);
-
-	/*
-	 * frequent_use here indicates this request comes from the arena bins,
-	 * in which case it will be split into slabs, and therefore there is no
-	 * intrinsic slack in the allocation (the entire range of allocated size
-	 * will be accessed).
-	 *
-	 * In this case bypass the slab_max_alloc limit (if still within the
-	 * huge page size).  These requests do not concern internal
-	 * fragmentation with huge pages (again, the full size will be used).
-	 */
-	if (!(frequent_reuse && size <= HUGEPAGE)
-	    && (size > shard->opts.slab_max_alloc)) {
-		return 0;
-	}
-
-	size_t nsuccess = hpa_alloc_batch_psset(
-	    tsdn, shard, size, nallocs, results, deferred_work_generated);
-
-	witness_assert_depth_to_rank(
-	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
-
 	/*
 	 * Guard the sanity checks with config_debug because the loop cannot be
 	 * proven non-circular by the compiler, even if everything within the
@@ -876,7 +852,6 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
 			assert(edata_base_get(edata) != NULL);
 		}
 	}
-	return nsuccess;
 }
 
 static edata_t *
@@ -891,16 +866,52 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
 	if (alignment > PAGE || zero) {
 		return NULL;
 	}
+	hpa_shard_t *shard = hpa_from_pai(self);
+
 	/*
-	 * An alloc with alignment == PAGE and zero == false is equivalent to a
+	 * frequent_use here indicates this request comes from the arena bins,
-	 * batch alloc of 1.  Just do that, so we can share code.
+	 * in which case it will be split into slabs, and therefore there is no
+	 * intrinsic slack in the allocation (the entire range of allocated size
+	 * will be accessed).
+	 *
+	 * In this case bypass the slab_max_alloc limit (if still within the
+	 * huge page size).  These requests do not concern internal
+	 * fragmentation with huge pages (again, the full size will be used).
 	 */
+	if (!(frequent_reuse && size <= HUGEPAGE)
+	    && (size > shard->opts.slab_max_alloc)) {
+		return NULL;
+	}
+	edata_t *edata = sec_alloc(tsdn, &shard->sec, size);
+	if (edata != NULL) {
+		return edata;
+	}
+	size_t              nallocs = sec_size_supported(&shard->sec, size)
+	                 ? shard->sec.opts.batch_fill_extra + 1
+	                 : 1;
 	edata_list_active_t results;
 	edata_list_active_init(&results);
-	size_t nallocs = hpa_alloc_batch(tsdn, self, size, /* nallocs */ 1,
+	size_t nsuccess = hpa_alloc_batch_psset(
-	    &results, frequent_reuse, deferred_work_generated);
+	    tsdn, shard, size, nallocs, &results, deferred_work_generated);
-	assert(nallocs == 0 || nallocs == 1);
+	hpa_assert_results(tsdn, shard, &results);
-	edata_t *edata = edata_list_active_first(&results);
+	edata = edata_list_active_first(&results);
+
+	if (edata != NULL) {
+		edata_list_active_remove(&results, edata);
+		assert(nsuccess > 0);
+		nsuccess--;
+	}
+	if (nsuccess > 0) {
+		assert(sec_size_supported(&shard->sec, size));
+		sec_fill(tsdn, &shard->sec, size, &results, nsuccess);
+		/* Unlikely rollback in case of overfill */
+		if (!edata_list_active_empty(&results)) {
+			hpa_dalloc_batch(
+			    tsdn, self, &results, deferred_work_generated);
+		}
+	}
+	witness_assert_depth_to_rank(
+	    tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
 	return edata;
 }
 
@@ -996,10 +1007,19 @@ static void
 hpa_dalloc(
     tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
 	assert(!edata_guarded_get(edata));
-	/* Just a dalloc_batch of size 1; this lets us share logic. */
+
 	edata_list_active_t dalloc_list;
 	edata_list_active_init(&dalloc_list);
 	edata_list_active_append(&dalloc_list, edata);
+
+	hpa_shard_t *shard = hpa_from_pai(self);
+	sec_dalloc(tsdn, &shard->sec, &dalloc_list);
+	if (edata_list_active_empty(&dalloc_list)) {
+		/* sec consumed the pointer */
+		*deferred_work_generated = false;
+		return;
+	}
+	/* We may have more than one pointer to flush now */
 	hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
 }
 
@@ -1063,15 +1083,32 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
 	return time_ns;
 }
 
+static void
+hpa_sec_flush_impl(tsdn_t *tsdn, hpa_shard_t *shard) {
+	edata_list_active_t to_flush;
+	edata_list_active_init(&to_flush);
+
+	sec_flush(tsdn, &shard->sec, &to_flush);
+	bool deferred_work_generated;
+	hpa_dalloc_batch(
+	    tsdn, (pai_t *)shard, &to_flush, &deferred_work_generated);
+}
+
 void
 hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) {
 	hpa_do_consistency_checks(shard);
+	hpa_sec_flush_impl(tsdn, shard);
 
 	malloc_mutex_lock(tsdn, &shard->mtx);
 	edata_cache_fast_disable(tsdn, &shard->ecf);
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 }
 
+void
+hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard) {
+	hpa_sec_flush_impl(tsdn, shard);
+}
+
 static void
 hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
 	assert(bin_stats->npageslabs == 0);
@@ -1093,6 +1130,7 @@ hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
 void
 hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
 	hpa_do_consistency_checks(shard);
+	hpa_shard_flush(tsdn, shard);
 	/*
 	 * By the time we're here, the arena code should have dalloc'd all the
 	 * active extents, which means we should have eventually evicted
@@ -1137,6 +1175,12 @@ hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
 	malloc_mutex_unlock(tsdn, &shard->mtx);
 }
 
+void
+hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard) {
+	hpa_do_consistency_checks(shard);
+	sec_prefork2(tsdn, &shard->sec);
+}
+
 void
 hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard) {
 	hpa_do_consistency_checks(shard);
@@ -1155,6 +1199,7 @@ void
 hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard) {
 	hpa_do_consistency_checks(shard);
 
+	sec_postfork_parent(tsdn, &shard->sec);
 	malloc_mutex_postfork_parent(tsdn, &shard->grow_mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->mtx);
 }
@@ -1163,6 +1208,7 @@ void
 hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard) {
 	hpa_do_consistency_checks(shard);
 
+	sec_postfork_child(tsdn, &shard->sec);
 	malloc_mutex_postfork_child(tsdn, &shard->grow_mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->mtx);
 }

src/jemalloc.c

@@ -1013,6 +1013,15 @@ malloc_conf_error(
 		/* However, tolerate experimental features. */
 		return;
 	}
+	const char  *deprecated[] = {"hpa_sec_bytes_after_flush"};
+	const size_t deprecated_cnt = (sizeof(deprecated)
+	    / sizeof(deprecated[0]));
+	for (size_t i = 0; i < deprecated_cnt; ++i) {
+		if (strncmp(k, deprecated[i], strlen(deprecated[i])) == 0) {
+			/* Tolerate deprecated features. */
+			return;
+		}
+	}
 	had_conf_error = true;
 }
 
@@ -1685,7 +1694,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 				}
 				CONF_CONTINUE;
 			}
-
 			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards,
 			    "hpa_sec_nshards", 0, 0, CONF_CHECK_MIN,
 			    CONF_DONT_CHECK_MAX, true);
@@ -1694,13 +1702,10 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			    USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN,
 			    CONF_CHECK_MAX, true);
 			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
-			    "hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN,
+			    "hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
-			    CONF_DONT_CHECK_MAX, true);
-			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush,
-			    "hpa_sec_bytes_after_flush", PAGE, 0,
 			    CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
 			CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
-			    "hpa_sec_batch_fill_extra", 0, HUGEPAGE_PAGES,
+			    "hpa_sec_batch_fill_extra", 1, HUGEPAGE_PAGES,
 			    CONF_CHECK_MIN, CONF_CHECK_MAX, true);
 
 			if (CONF_MATCH("slab_sizes")) {

src/pa.c

@@ -67,12 +67,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
 bool
 pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
     const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
-	if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap,
+	if (hpa_shard_init(tsdn, &shard->hpa_shard, &shard->central->hpa,
-	        shard->base, &shard->edata_cache, shard->ind, hpa_opts)) {
+	        shard->emap, shard->base, &shard->edata_cache, shard->ind,
-		return true;
+	        hpa_opts, hpa_sec_opts)) {
-	}
-	if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,
-	        hpa_sec_opts)) {
 		return true;
 	}
 	shard->ever_used_hpa = true;
@@ -85,7 +82,6 @@ void
 pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
 	atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
 	if (shard->ever_used_hpa) {
-		sec_disable(tsdn, &shard->hpa_sec);
 		hpa_shard_disable(tsdn, &shard->hpa_shard);
 	}
 }
@@ -93,8 +89,13 @@ pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
 	atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
+	pa_shard_flush(tsdn, shard);
+}
+
+void
+pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (shard->ever_used_hpa) {
-		sec_flush(tsdn, &shard->hpa_sec);
+		hpa_shard_flush(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -107,7 +108,6 @@ void
 pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
 	pac_destroy(tsdn, &shard->pac);
 	if (shard->ever_used_hpa) {
-		sec_flush(tsdn, &shard->hpa_sec);
 		hpa_shard_destroy(tsdn, &shard->hpa_shard);
 	}
 }
@@ -115,7 +115,7 @@ pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
 static pai_t *
 pa_get_pai(pa_shard_t *shard, edata_t *edata) {
 	return (edata_pai_get(edata) == EXTENT_PAI_PAC ? &shard->pac.pai
-	                                               : &shard->hpa_sec.pai);
+	                                               : &shard->hpa_shard.pai);
 }
 
 edata_t *
@@ -128,7 +128,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
 
 	edata_t *edata = NULL;
 	if (!guarded && pa_shard_uses_hpa(shard)) {
-		edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment,
+		edata = pai_alloc(tsdn, &shard->hpa_shard.pai, size, alignment,
 		    zero, /* guarded */ false, slab, deferred_work_generated);
 	}
 	/*

src/pa_extra.c

@@ -17,7 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
 void
 pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
 	if (shard->ever_used_hpa) {
-		sec_prefork2(tsdn, &shard->hpa_sec);
+		hpa_shard_prefork2(tsdn, &shard->hpa_shard);
 	}
 }
 
@@ -54,7 +54,6 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
-		sec_postfork_parent(tsdn, &shard->hpa_sec);
 		hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
 	}
 }
@@ -69,7 +68,6 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
 	malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
 	if (shard->ever_used_hpa) {
-		sec_postfork_child(tsdn, &shard->hpa_sec);
 		hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
 	}
 }
@@ -104,8 +102,7 @@ pa_shard_basic_stats_merge(
 void
 pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
     pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
-    hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
+    hpa_shard_stats_t *hpa_stats_out, size_t *resident) {
-    size_t *resident) {
 	cassert(config_stats);
 
 	pa_shard_stats_out->pac_stats.retained +=
@@ -170,7 +167,6 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
 
 	if (shard->ever_used_hpa) {
 		hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
-		sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out);
 	}
 }
 
@@ -204,7 +200,7 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
 		pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
 		    &shard->hpa_shard.grow_mtx,
 		    arena_prof_mutex_hpa_shard_grow);
-		sec_mutex_stats_read(tsdn, &shard->hpa_sec,
+		sec_mutex_stats_read(tsdn, &shard->hpa_shard.sec,
 		    &mutex_prof_data[arena_prof_mutex_hpa_sec]);
 	}
 }

src/pac.c

@@ -97,11 +97,9 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
 	atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
 
 	pac->pai.alloc = &pac_alloc_impl;
-	pac->pai.alloc_batch = &pai_alloc_batch_default;
 	pac->pai.expand = &pac_expand_impl;
 	pac->pai.shrink = &pac_shrink_impl;
 	pac->pai.dalloc = &pac_dalloc_impl;
-	pac->pai.dalloc_batch = &pai_dalloc_batch_default;
 	pac->pai.time_until_deferred_work = &pac_time_until_deferred_work;
 
 	return false;
@@ -449,8 +447,8 @@ decay_with_process_madvise(edata_list_inactive_t *decay_extents) {
 
 	size_t cur = 0, total_bytes = 0;
 	for (edata_t *edata = edata_list_inactive_first(decay_extents);
-	     edata != NULL;
+	    edata != NULL;
-	     edata = edata_list_inactive_next(decay_extents, edata)) {
+	    edata = edata_list_inactive_next(decay_extents, edata)) {
 		size_t pages_bytes = edata_size_get(edata);
 		vec[cur].iov_base = edata_base_get(edata);
 		vec[cur].iov_len = pages_bytes;
@@ -511,7 +509,7 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
 	}
 
 	for (edata_t *edata = edata_list_inactive_first(decay_extents);
-	     edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
+	    edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
 		edata_list_inactive_remove(decay_extents, edata);
 
 		size_t size = edata_size_get(edata);

src/pai.c

@@ -1,32 +0,0 @@
-#include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/jemalloc_internal_includes.h"
-
-size_t
-pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
-    edata_list_active_t *results, bool frequent_reuse,
-    bool *deferred_work_generated) {
-	for (size_t i = 0; i < nallocs; i++) {
-		bool     deferred_by_alloc = false;
-		edata_t *edata = pai_alloc(tsdn, self, size, PAGE,
-		    /* zero */ false, /* guarded */ false, frequent_reuse,
-		    &deferred_by_alloc);
-		*deferred_work_generated |= deferred_by_alloc;
-		if (edata == NULL) {
-			return i;
-		}
-		edata_list_active_append(results, edata);
-	}
-	return nallocs;
-}
-
-void
-pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
-    bool *deferred_work_generated) {
-	edata_t *edata;
-	while ((edata = edata_list_active_first(list)) != NULL) {
-		bool deferred_by_dalloc = false;
-		edata_list_active_remove(list, edata);
-		pai_dalloc(tsdn, self, edata, &deferred_by_dalloc);
-		*deferred_work_generated |= deferred_by_dalloc;
-	}
-}

src/sec.c

@@ -4,95 +4,56 @@
 #include "jemalloc/internal/sec.h"
 #include "jemalloc/internal/jemalloc_probe.h"
 
-static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
+static bool
-    size_t alignment, bool zero, bool guarded, bool frequent_reuse,
-    bool *deferred_work_generated);
-static bool     sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-        size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
-static bool     sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
-        size_t old_size, size_t new_size, bool *deferred_work_generated);
-static void     sec_dalloc(
-        tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
-
-static void
 sec_bin_init(sec_bin_t *bin) {
-	bin->being_batch_filled = false;
 	bin->bytes_cur = 0;
+	sec_bin_stats_init(&bin->stats);
 	edata_list_active_init(&bin->freelist);
+	bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN,
+	    malloc_mutex_rank_exclusive);
+	if (err) {
+		return true;
+	}
+
+	return false;
 }
 
 bool
-sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
+sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
-    const sec_opts_t *opts) {
+	sec->opts = *opts;
+	if (opts->nshards == 0) {
+		return false;
+	}
 	assert(opts->max_alloc >= PAGE);
+
 	/*
 	 * Same as tcache, sec do not cache allocs/dallocs larger than
 	 * USIZE_GROW_SLOW_THRESHOLD because the usize above this increases
 	 * by PAGE and the number of usizes is too large.
 	 */
-	assert(!sz_large_size_classes_disabled()
+	assert(opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
-	    || opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
 
 	size_t   max_alloc = PAGE_FLOOR(opts->max_alloc);
 	pszind_t npsizes = sz_psz2ind(max_alloc) + 1;
 
-	size_t sz_shards = opts->nshards * sizeof(sec_shard_t);
+	size_t ntotal_bins = opts->nshards * (size_t)npsizes;
-	size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t);
+	size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
-	size_t sz_alloc = sz_shards + sz_bins;
+	void  *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
-	void  *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE);
 	if (dynalloc == NULL) {
 		return true;
 	}
-	sec_shard_t *shard_cur = (sec_shard_t *)dynalloc;
+	sec->bins = (sec_bin_t *)dynalloc;
-	sec->shards = shard_cur;
+	for (pszind_t j = 0; j < ntotal_bins; j++) {
-	sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards];
+		if (sec_bin_init(&sec->bins[j])) {
-	/* Just for asserts, below. */
-	sec_bin_t *bin_start = bin_cur;
-
-	for (size_t i = 0; i < opts->nshards; i++) {
-		sec_shard_t *shard = shard_cur;
-		shard_cur++;
-		bool err = malloc_mutex_init(&shard->mtx, "sec_shard",
-		    WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive);
-		if (err) {
 			return true;
 		}
-		shard->enabled = true;
-		shard->bins = bin_cur;
-		for (pszind_t j = 0; j < npsizes; j++) {
-			sec_bin_init(&shard->bins[j]);
-			bin_cur++;
-		}
-		shard->bytes_cur = 0;
-		shard->to_flush_next = 0;
 	}
-	/*
-	 * Should have exactly matched the bin_start to the first unused byte
-	 * after the shards.
-	 */
-	assert((void *)shard_cur == (void *)bin_start);
-	/* And the last bin to use up the last bytes of the allocation. */
-	assert((char *)bin_cur == ((char *)dynalloc + sz_alloc));
-	sec->fallback = fallback;
-
-	sec->opts = *opts;
 	sec->npsizes = npsizes;
 
-	/*
-	 * Initialize these last so that an improper use of an SEC whose
-	 * initialization failed will segfault in an easy-to-spot way.
-	 */
-	sec->pai.alloc = &sec_alloc;
-	sec->pai.alloc_batch = &pai_alloc_batch_default;
-	sec->pai.expand = &sec_expand;
-	sec->pai.shrink = &sec_shrink;
-	sec->pai.dalloc = &sec_dalloc;
-	sec->pai.dalloc_batch = &pai_dalloc_batch_default;
-
 	return false;
 }
 
-static sec_shard_t *
+static uint8_t
 sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
 	/*
 	 * Eventually, we should implement affinity, tracking source shard using
@@ -100,7 +61,7 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
 	 * distribute across all shards.
 	 */
 	if (tsdn_null(tsdn)) {
-		return &sec->shards[0];
+		return 0;
 	}
 	tsd_t   *tsd = tsdn_tsd(tsdn);
 	uint8_t *idxp = tsd_sec_shardp_get(tsd);
@@ -118,284 +79,252 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
 		assert(idx < (uint32_t)sec->opts.nshards);
 		*idxp = (uint8_t)idx;
 	}
-	return &sec->shards[*idxp];
+	return *idxp;
 }
 
-/*
+static sec_bin_t *
- * Perhaps surprisingly, this can be called on the alloc pathways; if we hit an
+sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) {
- * empty cache, we'll try to fill it, which can push the shard over it's limit.
+	assert(shard < sec->opts.nshards);
- */
+	size_t ind = (size_t)shard * sec->npsizes + pszind;
-static void
+	assert(ind < sec->npsizes * sec->opts.nshards);
-sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
+	return &sec->bins[ind];
-	malloc_mutex_assert_owner(tsdn, &shard->mtx);
-	edata_list_active_t to_flush;
-	edata_list_active_init(&to_flush);
-	while (shard->bytes_cur > sec->opts.bytes_after_flush) {
-		/* Pick a victim. */
-		sec_bin_t *bin = &shard->bins[shard->to_flush_next];
-
-		/* Update our victim-picking state. */
-		shard->to_flush_next++;
-		if (shard->to_flush_next == sec->npsizes) {
-			shard->to_flush_next = 0;
-		}
-
-		assert(shard->bytes_cur >= bin->bytes_cur);
-		if (bin->bytes_cur != 0) {
-			shard->bytes_cur -= bin->bytes_cur;
-			bin->bytes_cur = 0;
-			edata_list_active_concat(&to_flush, &bin->freelist);
-		}
-		/*
-		 * Either bin->bytes_cur was 0, in which case we didn't touch
-		 * the bin list but it should be empty anyways (or else we
-		 * missed a bytes_cur update on a list modification), or it
-		 * *was* 0 and we emptied it ourselves.  Either way, it should
-		 * be empty now.
-		 */
-		assert(edata_list_active_empty(&bin->freelist));
-	}
-
-	malloc_mutex_unlock(tsdn, &shard->mtx);
-	bool deferred_work_generated = false;
-	pai_dalloc_batch(
-	    tsdn, sec->fallback, &to_flush, &deferred_work_generated);
 }
 
 static edata_t *
-sec_shard_alloc_locked(
+sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
-    tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, sec_bin_t *bin) {
+	malloc_mutex_assert_owner(tsdn, &bin->mtx);
-	malloc_mutex_assert_owner(tsdn, &shard->mtx);
+
-	if (!shard->enabled) {
-		return NULL;
-	}
 	edata_t *edata = edata_list_active_first(&bin->freelist);
 	if (edata != NULL) {
+		assert(!edata_list_active_empty(&bin->freelist));
 		edata_list_active_remove(&bin->freelist, edata);
-		assert(edata_size_get(edata) <= bin->bytes_cur);
+		size_t sz = edata_size_get(edata);
-		bin->bytes_cur -= edata_size_get(edata);
+		assert(sz <= bin->bytes_cur && sz > 0);
-		assert(edata_size_get(edata) <= shard->bytes_cur);
+		bin->bytes_cur -= sz;
-		shard->bytes_cur -= edata_size_get(edata);
+		bin->stats.nhits++;
 	}
 	return edata;
 }
 
 static edata_t *
-sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
+sec_multishard_trylock_alloc(
-    sec_bin_t *bin, size_t size, bool frequent_reuse) {
+    tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
-	malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
+	assert(sec->opts.nshards > 0);
 
-	edata_list_active_t result;
+	uint8_t    cur_shard = sec_shard_pick(tsdn, sec);
-	edata_list_active_init(&result);
+	sec_bin_t *bin;
-	bool   deferred_work_generated = false;
+	for (size_t i = 0; i < sec->opts.nshards; ++i) {
-	size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size,
+		bin = sec_bin_pick(sec, cur_shard, pszind);
-	    1 + sec->opts.batch_fill_extra, &result, frequent_reuse,
+		if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
-	    &deferred_work_generated);
+			edata_t *edata = sec_bin_alloc_locked(
-
+			    tsdn, sec, bin, size);
-	edata_t *ret = edata_list_active_first(&result);
+			malloc_mutex_unlock(tsdn, &bin->mtx);
-	if (ret != NULL) {
+			if (edata != NULL) {
-		edata_list_active_remove(&result, ret);
+				JE_USDT(sec_alloc, 5, sec, bin, edata, size,
+				    /* frequent_reuse */ 1);
+				return edata;
+			}
+		}
+		cur_shard++;
+		if (cur_shard == sec->opts.nshards) {
+			cur_shard = 0;
+		}
 	}
-
+	/* No bin had alloc or had the extent */
-	malloc_mutex_lock(tsdn, &shard->mtx);
+	assert(cur_shard == sec_shard_pick(tsdn, sec));
-	bin->being_batch_filled = false;
+	bin = sec_bin_pick(sec, cur_shard, pszind);
-	/*
+	malloc_mutex_lock(tsdn, &bin->mtx);
-	 * Handle the easy case first: nothing to cache.  Note that this can
+	edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
-	 * only happen in case of OOM, since sec_alloc checks the expected
+	if (edata == NULL) {
-	 * number of allocs, and doesn't bother going down the batch_fill
+		/* Only now we know it is a miss */
-	 * pathway if there won't be anything left to cache.  So to be in this
+		bin->stats.nmisses++;
-	 * code path, we must have asked for > 1 alloc, but only gotten 1 back.
-	 */
-	if (nalloc <= 1) {
-		malloc_mutex_unlock(tsdn, &shard->mtx);
-		return ret;
 	}
-
+	malloc_mutex_unlock(tsdn, &bin->mtx);
-	size_t new_cached_bytes = (nalloc - 1) * size;
+	JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1);
-
+	return edata;
-	edata_list_active_concat(&bin->freelist, &result);
-	bin->bytes_cur += new_cached_bytes;
-	shard->bytes_cur += new_cached_bytes;
-
-	if (shard->bytes_cur > sec->opts.max_bytes) {
-		sec_flush_some_and_unlock(tsdn, sec, shard);
-	} else {
-		malloc_mutex_unlock(tsdn, &shard->mtx);
-	}
-
-	return ret;
 }
 
-static edata_t *
+edata_t *
-sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
+sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
-    bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
+	if (!sec_size_supported(sec, size)) {
+		return NULL;
+	}
 	assert((size & PAGE_MASK) == 0);
-	assert(!guarded);
-
-	sec_t *sec = (sec_t *)self;
-
-	if (zero || alignment > PAGE || sec->opts.nshards == 0
-	    || size > sec->opts.max_alloc) {
-		return pai_alloc(tsdn, sec->fallback, size, alignment, zero,
-		    /* guarded */ false, frequent_reuse,
-		    deferred_work_generated);
-	}
 	pszind_t pszind = sz_psz2ind(size);
 	assert(pszind < sec->npsizes);
 
-	sec_shard_t *shard = sec_shard_pick(tsdn, sec);
+	/*
-	sec_bin_t   *bin = &shard->bins[pszind];
+	 * If there's only one shard, skip the trylock optimization and
-	bool         do_batch_fill = false;
+	 * go straight to the blocking lock.
-
+	 */
-	malloc_mutex_lock(tsdn, &shard->mtx);
+	if (sec->opts.nshards == 1) {
-	edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin);
+		sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
-	if (edata == NULL) {
+		malloc_mutex_lock(tsdn, &bin->mtx);
-		if (!bin->being_batch_filled
+		edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
-		    && sec->opts.batch_fill_extra > 0) {
+		if (edata == NULL) {
-			bin->being_batch_filled = true;
+			bin->stats.nmisses++;
-			do_batch_fill = true;
 		}
+		malloc_mutex_unlock(tsdn, &bin->mtx);
+		JE_USDT(sec_alloc, 5, sec, bin, edata, size,
+		    /* frequent_reuse */ 1);
+		return edata;
 	}
-	malloc_mutex_unlock(tsdn, &shard->mtx);
+	return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
-	if (edata == NULL) {
-		if (do_batch_fill) {
-			edata = sec_batch_fill_and_alloc(
-			    tsdn, sec, shard, bin, size, frequent_reuse);
-		} else {
-			edata = pai_alloc(tsdn, sec->fallback, size, alignment,
-			    zero, /* guarded */ false, frequent_reuse,
-			    deferred_work_generated);
-		}
-	}
-	JE_USDT(sec_alloc, 5, sec, shard, edata, size, frequent_reuse);
-	return edata;
-}
-
-static bool
-sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size, bool zero, bool *deferred_work_generated) {
-	sec_t *sec = (sec_t *)self;
-	JE_USDT(sec_expand, 4, sec, edata, old_size, new_size);
-	return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero,
-	    deferred_work_generated);
-}
-
-static bool
-sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
-    size_t new_size, bool *deferred_work_generated) {
-	sec_t *sec = (sec_t *)self;
-	JE_USDT(sec_shrink, 4, sec, edata, old_size, new_size);
-	return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size,
-	    deferred_work_generated);
 }
 
 static void
-sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
+sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
-	malloc_mutex_assert_owner(tsdn, &shard->mtx);
+    edata_list_active_t *dalloc_list) {
-	shard->bytes_cur = 0;
+	malloc_mutex_assert_owner(tsdn, &bin->mtx);
-	edata_list_active_t to_flush;
-	edata_list_active_init(&to_flush);
-	for (pszind_t i = 0; i < sec->npsizes; i++) {
-		sec_bin_t *bin = &shard->bins[i];
-		bin->bytes_cur = 0;
-		edata_list_active_concat(&to_flush, &bin->freelist);
-	}
 
-	/*
-	 * Ordinarily we would try to avoid doing the batch deallocation while
-	 * holding the shard mutex, but the flush_all pathways only happen when
-	 * we're disabling the HPA or resetting the arena, both of which are
-	 * rare pathways.
-	 */
-	bool deferred_work_generated = false;
-	pai_dalloc_batch(
-	    tsdn, sec->fallback, &to_flush, &deferred_work_generated);
-}
-
-static void
-sec_shard_dalloc_and_unlock(
-    tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_t *edata) {
-	malloc_mutex_assert_owner(tsdn, &shard->mtx);
-	assert(shard->bytes_cur <= sec->opts.max_bytes);
-	size_t   size = edata_size_get(edata);
-	pszind_t pszind = sz_psz2ind(size);
-	assert(pszind < sec->npsizes);
-	/*
-	 * Prepending here results in LIFO allocation per bin, which seems
-	 * reasonable.
-	 */
-	sec_bin_t *bin = &shard->bins[pszind];
-	edata_list_active_prepend(&bin->freelist, edata);
 	bin->bytes_cur += size;
-	shard->bytes_cur += size;
+	edata_t *edata = edata_list_active_first(dalloc_list);
-	if (shard->bytes_cur > sec->opts.max_bytes) {
+	assert(edata != NULL);
-		/*
+	edata_list_active_remove(dalloc_list, edata);
-		 * We've exceeded the shard limit.  We make two nods in the
+	JE_USDT(sec_dalloc, 3, sec, bin, edata);
-		 * direction of fragmentation avoidance: we flush everything in
+	edata_list_active_prepend(&bin->freelist, edata);
-		 * the shard, rather than one particular bin, and we hold the
+	/* Single extent can be returned to SEC */
-		 * lock while flushing (in case one of the extents we flush is
+	assert(edata_list_active_empty(dalloc_list));
-		 * highly preferred from a fragmentation-avoidance perspective
-		 * in the backing allocator).  This has the extra advantage of
-		 * not requiring advanced cache balancing strategies.
-		 */
-		sec_flush_some_and_unlock(tsdn, sec, shard);
-		malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
-	} else {
-		malloc_mutex_unlock(tsdn, &shard->mtx);
-	}
-}
 
-static void
+	if (bin->bytes_cur <= sec->opts.max_bytes) {
-sec_dalloc(
+		bin->stats.ndalloc_noflush++;
-    tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
-	sec_t *sec = (sec_t *)self;
-	if (sec->opts.nshards == 0
-	    || edata_size_get(edata) > sec->opts.max_alloc) {
-		pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
 		return;
 	}
-	sec_shard_t *shard = sec_shard_pick(tsdn, sec);
+	bin->stats.ndalloc_flush++;
-	JE_USDT(sec_dalloc, 3, sec, shard, edata);
+	/* we want to flush 1/4 of max_bytes */
-	malloc_mutex_lock(tsdn, &shard->mtx);
+	size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2);
-	if (shard->enabled) {
+	while (bin->bytes_cur > bytes_target
-		sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata);
+	    && !edata_list_active_empty(&bin->freelist)) {
+		edata_t *cur = edata_list_active_last(&bin->freelist);
+		size_t   sz = edata_size_get(cur);
+		assert(sz <= bin->bytes_cur && sz > 0);
+		bin->bytes_cur -= sz;
+		edata_list_active_remove(&bin->freelist, cur);
+		edata_list_active_append(dalloc_list, cur);
+	}
+}
+
+static void
+sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
+    pszind_t pszind, edata_list_active_t *dalloc_list) {
+	assert(sec->opts.nshards > 0);
+
+	/* Try to dalloc in this threads bin first */
+	uint8_t cur_shard = sec_shard_pick(tsdn, sec);
+	for (size_t i = 0; i < sec->opts.nshards; ++i) {
+		sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
+		if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
+			sec_bin_dalloc_locked(
+			    tsdn, sec, bin, size, dalloc_list);
+			malloc_mutex_unlock(tsdn, &bin->mtx);
+			return;
+		}
+		cur_shard++;
+		if (cur_shard == sec->opts.nshards) {
+			cur_shard = 0;
+		}
+	}
+	/* No bin had alloc or had the extent */
+	assert(cur_shard == sec_shard_pick(tsdn, sec));
+	sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
+	malloc_mutex_lock(tsdn, &bin->mtx);
+	sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
+	malloc_mutex_unlock(tsdn, &bin->mtx);
+}
+
+void
+sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
+	if (!sec_is_used(sec)) {
+		return;
+	}
+	edata_t *edata = edata_list_active_first(dalloc_list);
+	size_t   size = edata_size_get(edata);
+	if (size > sec->opts.max_alloc) {
+		return;
+	}
+	pszind_t pszind = sz_psz2ind(size);
+	assert(pszind < sec->npsizes);
+
+	/*
+         * If there's only one shard, skip the trylock optimization and
+	 * go straight to the blocking lock.
+	 */
+	if (sec->opts.nshards == 1) {
+		sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
+		malloc_mutex_lock(tsdn, &bin->mtx);
+		sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
+		malloc_mutex_unlock(tsdn, &bin->mtx);
+		return;
+	}
+	sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
+}
+
+void
+sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
+    size_t nallocs) {
+	assert((size & PAGE_MASK) == 0);
+	assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
+	assert(nallocs > 0);
+
+	pszind_t pszind = sz_psz2ind(size);
+	assert(pszind < sec->npsizes);
+
+	sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
+	malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
+	malloc_mutex_lock(tsdn, &bin->mtx);
+	size_t new_cached_bytes = nallocs * size;
+	if (bin->bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
+		assert(!edata_list_active_empty(result));
+		edata_list_active_concat(&bin->freelist, result);
+		bin->bytes_cur += new_cached_bytes;
 	} else {
-		malloc_mutex_unlock(tsdn, &shard->mtx);
+		/*
-		pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
+		 * Unlikely case of many threads filling at the same time and
+		 * going above max.
+		 */
+		bin->stats.noverfills++;
+		while (bin->bytes_cur + size <= sec->opts.max_bytes) {
+			edata_t *edata = edata_list_active_first(result);
+			if (edata == NULL) {
+				break;
+			}
+			edata_list_active_remove(result, edata);
+			assert(size == edata_size_get(edata));
+			edata_list_active_append(&bin->freelist, edata);
+			bin->bytes_cur += size;
+		}
 	}
+	malloc_mutex_unlock(tsdn, &bin->mtx);
 }
 
 void
-sec_flush(tsdn_t *tsdn, sec_t *sec) {
+sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
-	for (size_t i = 0; i < sec->opts.nshards; i++) {
+	if (!sec_is_used(sec)) {
-		malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
+		return;
-		sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
-		malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
 	}
-}
+	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
-
+	for (pszind_t i = 0; i < ntotal_bins; i++) {
-void
+		sec_bin_t *bin = &sec->bins[i];
-sec_disable(tsdn_t *tsdn, sec_t *sec) {
+		malloc_mutex_lock(tsdn, &bin->mtx);
-	for (size_t i = 0; i < sec->opts.nshards; i++) {
+		bin->bytes_cur = 0;
-		malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
+		edata_list_active_concat(to_flush, &bin->freelist);
-		sec->shards[i].enabled = false;
+		malloc_mutex_unlock(tsdn, &bin->mtx);
-		sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
-		malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
 	}
 }
 
 void
 sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
+	if (!sec_is_used(sec)) {
+		return;
+	}
 	size_t sum = 0;
-	for (size_t i = 0; i < sec->opts.nshards; i++) {
+	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
-		/*
+	for (pszind_t i = 0; i < ntotal_bins; i++) {
-		 * We could save these lock acquisitions by making bytes_cur
+		sec_bin_t *bin = &sec->bins[i];
-		 * atomic, but stats collection is rare anyways and we expect
+		malloc_mutex_lock(tsdn, &bin->mtx);
-		 * the number and type of stats to get more interesting.
+		sum += bin->bytes_cur;
-		 */
+		sec_bin_stats_accum(&stats->total, &bin->stats);
-		malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
+		malloc_mutex_unlock(tsdn, &bin->mtx);
-		sum += sec->shards[i].bytes_cur;
-		malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
 	}
 	stats->bytes += sum;
 }
@@ -403,31 +332,50 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
 void
 sec_mutex_stats_read(
     tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data) {
-	for (size_t i = 0; i < sec->opts.nshards; i++) {
+	if (!sec_is_used(sec)) {
-		malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
+		return;
-		malloc_mutex_prof_accum(
+	}
-		    tsdn, mutex_prof_data, &sec->shards[i].mtx);
+	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
-		malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
+	for (pszind_t i = 0; i < ntotal_bins; i++) {
+		sec_bin_t *bin = &sec->bins[i];
+		malloc_mutex_lock(tsdn, &bin->mtx);
+		malloc_mutex_prof_accum(tsdn, mutex_prof_data, &bin->mtx);
+		malloc_mutex_unlock(tsdn, &bin->mtx);
 	}
 }
 
 void
 sec_prefork2(tsdn_t *tsdn, sec_t *sec) {
-	for (size_t i = 0; i < sec->opts.nshards; i++) {
+	if (!sec_is_used(sec)) {
-		malloc_mutex_prefork(tsdn, &sec->shards[i].mtx);
+		return;
+	}
+	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
+	for (pszind_t i = 0; i < ntotal_bins; i++) {
+		sec_bin_t *bin = &sec->bins[i];
+		malloc_mutex_prefork(tsdn, &bin->mtx);
 	}
 }
 
 void
 sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) {
-	for (size_t i = 0; i < sec->opts.nshards; i++) {
+	if (!sec_is_used(sec)) {
-		malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx);
+		return;
+	}
+	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
+	for (pszind_t i = 0; i < ntotal_bins; i++) {
+		sec_bin_t *bin = &sec->bins[i];
+		malloc_mutex_postfork_parent(tsdn, &bin->mtx);
 	}
 }
 
 void
 sec_postfork_child(tsdn_t *tsdn, sec_t *sec) {
-	for (size_t i = 0; i < sec->opts.nshards; i++) {
+	if (!sec_is_used(sec)) {
-		malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx);
+		return;
+	}
+	size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
+	for (pszind_t i = 0; i < ntotal_bins; i++) {
+		sec_bin_t *bin = &sec->bins[i];
+		malloc_mutex_postfork_child(tsdn, &bin->mtx);
 	}
 }

src/stats.c

@@ -791,9 +791,35 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
 static void
 stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) {
 	size_t sec_bytes;
+	size_t sec_hits;
+	size_t sec_misses;
+	size_t sec_dalloc_flush;
+	size_t sec_dalloc_noflush;
+	size_t sec_overfills;
 	CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t);
 	emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache",
 	    emitter_type_size, &sec_bytes);
+	CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec_hits, size_t);
+	emitter_kv(emitter, "sec_hits", "Total hits in small extent cache",
+	    emitter_type_size, &sec_hits);
+	CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec_misses, size_t);
+	emitter_kv(emitter, "sec_misses", "Total misses in small extent cache",
+	    emitter_type_size, &sec_misses);
+	CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i,
+	    &sec_dalloc_noflush, size_t);
+	emitter_kv(emitter, "sec_dalloc_noflush",
+	    "Dalloc calls without flush in small extent cache",
+	    emitter_type_size, &sec_dalloc_noflush);
+	CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, &sec_dalloc_flush,
+	    size_t);
+	emitter_kv(emitter, "sec_dalloc_flush",
+	    "Dalloc calls with flush in small extent cache", emitter_type_size,
+	    &sec_dalloc_flush);
+	CTL_M2_GET(
+	    "stats.arenas.0.hpa_sec_overfills", i, &sec_overfills, size_t);
+	emitter_kv(emitter, "sec_overfills",
+	    "sec_fill calls that went over max_bytes", emitter_type_size,
+	    &sec_overfills);
 }
 
 static void
@@ -1642,7 +1668,6 @@ stats_general_print(emitter_t *emitter) {
 	OPT_WRITE_SIZE_T("hpa_sec_nshards")
 	OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
 	OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
-	OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush")
 	OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
 	OPT_WRITE_BOOL("huge_arena_pac_thp")
 	OPT_WRITE_CHAR_P("metadata_thp")

test/unit/hpa.c

@@ -113,10 +113,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
 
 	err = hpa_central_init(&test_data->central, test_data->base, hooks);
 	assert_false(err, "");
-
+	sec_opts_t sec_opts;
-	err = hpa_shard_init(&test_data->shard, &test_data->central,
+	sec_opts.nshards = 0;
+	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
+	err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
 	    &test_data->emap, test_data->base, &test_data->shard_edata_cache,
-	    SHARD_IND, opts);
+	    SHARD_IND, opts, &sec_opts);
 	assert_false(err, "");
 
 	return (hpa_shard_t *)test_data;
@@ -309,83 +311,6 @@ TEST_BEGIN(test_stress) {
 }
 TEST_END
 
-static void
-expect_contiguous(edata_t **edatas, size_t nedatas) {
-	for (size_t i = 0; i < nedatas; i++) {
-		size_t expected = (size_t)edata_base_get(edatas[0]) + i * PAGE;
-		expect_zu_eq(expected, (size_t)edata_base_get(edatas[i]),
-		    "Mismatch at index %zu", i);
-	}
-}
-
-TEST_BEGIN(test_alloc_dalloc_batch) {
-	test_skip_if(!hpa_supported());
-
-	hpa_shard_t *shard = create_test_data(
-	    &hpa_hooks_default, &test_hpa_shard_opts_default);
-	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
-
-	bool deferred_work_generated = false;
-
-	enum { NALLOCS = 8 };
-
-	edata_t *allocs[NALLOCS];
-	/*
-	 * Allocate a mix of ways; first half from regular alloc, second half
-	 * from alloc_batch.
-	 */
-	for (size_t i = 0; i < NALLOCS / 2; i++) {
-		allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE,
-		    /* zero */ false, /* guarded */ false,
-		    /* frequent_reuse */ false, &deferred_work_generated);
-		expect_ptr_not_null(allocs[i], "Unexpected alloc failure");
-	}
-	edata_list_active_t allocs_list;
-	edata_list_active_init(&allocs_list);
-	size_t nsuccess = pai_alloc_batch(tsdn, &shard->pai, PAGE, NALLOCS / 2,
-	    &allocs_list, /* frequent_reuse */ false, &deferred_work_generated);
-	expect_zu_eq(NALLOCS / 2, nsuccess, "Unexpected oom");
-	for (size_t i = NALLOCS / 2; i < NALLOCS; i++) {
-		allocs[i] = edata_list_active_first(&allocs_list);
-		edata_list_active_remove(&allocs_list, allocs[i]);
-	}
-
-	/*
-	 * Should have allocated them contiguously, despite the differing
-	 * methods used.
-	 */
-	void *orig_base = edata_base_get(allocs[0]);
-	expect_contiguous(allocs, NALLOCS);
-
-	/*
-	 * Batch dalloc the first half, individually deallocate the second half.
-	 */
-	for (size_t i = 0; i < NALLOCS / 2; i++) {
-		edata_list_active_append(&allocs_list, allocs[i]);
-	}
-	pai_dalloc_batch(
-	    tsdn, &shard->pai, &allocs_list, &deferred_work_generated);
-	for (size_t i = NALLOCS / 2; i < NALLOCS; i++) {
-		pai_dalloc(
-		    tsdn, &shard->pai, allocs[i], &deferred_work_generated);
-	}
-
-	/* Reallocate (individually), and ensure reuse and contiguity. */
-	for (size_t i = 0; i < NALLOCS; i++) {
-		allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE,
-		    /* zero */ false, /* guarded */ false, /* frequent_reuse */
-		    false, &deferred_work_generated);
-		expect_ptr_not_null(allocs[i], "Unexpected alloc failure.");
-	}
-	void *new_base = edata_base_get(allocs[0]);
-	expect_ptr_eq(
-	    orig_base, new_base, "Failed to reuse the allocated memory.");
-	expect_contiguous(allocs, NALLOCS);
-
-	destroy_test_data(shard);
-}
-TEST_END
-
 static uintptr_t defer_bump_ptr = HUGEPAGE * 123;
 static void *
 defer_test_map(size_t size) {
@@ -1533,8 +1458,7 @@ main(void) {
 	(void)mem_tree_iter;
 	(void)mem_tree_reverse_iter;
 	(void)mem_tree_destroy;
-	return test_no_reentrancy(test_alloc_max, test_stress,
+	return test_no_reentrancy(test_alloc_max, test_stress, test_defer_time,
-	    test_alloc_dalloc_batch, test_defer_time,
 	    test_purge_no_infinite_loop, test_no_min_purge_interval,
 	    test_min_purge_interval, test_purge,
 	    test_experimental_max_purge_nhp, test_vectorized_opt_eq_zero,

test/unit/hpa_sec_integration.c

@@ -0,0 +1,239 @@
+#include "test/jemalloc_test.h"
+
+#include "jemalloc/internal/hpa.h"
+#include "jemalloc/internal/nstime.h"
+
+#define SHARD_IND 111
+
+#define ALLOC_MAX (HUGEPAGE)
+
+typedef struct test_data_s test_data_t;
+struct test_data_s {
+	/*
+	 * Must be the first member -- we convert back and forth between the
+	 * test_data_t and the hpa_shard_t;
+	 */
+	hpa_shard_t   shard;
+	hpa_central_t central;
+	base_t       *base;
+	edata_cache_t shard_edata_cache;
+
+	emap_t emap;
+};
+
+static hpa_shard_opts_t test_hpa_shard_opts = {
+    /* slab_max_alloc */
+    HUGEPAGE,
+    /* hugification_threshold */
+    0.9 * HUGEPAGE,
+    /* dirty_mult */
+    FXP_INIT_PERCENT(10),
+    /* deferral_allowed */
+    true,
+    /* hugify_delay_ms */
+    0,
+    /* hugify_sync */
+    false,
+    /* min_purge_interval_ms */
+    5,
+    /* experimental_max_purge_nhp */
+    -1,
+    /* purge_threshold */
+    PAGE,
+    /* min_purge_delay_ms */
+    10,
+    /* hugify_style */
+    hpa_hugify_style_lazy};
+
+static hpa_shard_t *
+create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts,
+    const sec_opts_t *sec_opts) {
+	bool    err;
+	base_t *base = base_new(TSDN_NULL, /* ind */ SHARD_IND,
+	    &ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
+	assert_ptr_not_null(base, "");
+
+	test_data_t *test_data = malloc(sizeof(test_data_t));
+	assert_ptr_not_null(test_data, "");
+
+	test_data->base = base;
+
+	err = edata_cache_init(&test_data->shard_edata_cache, base);
+	assert_false(err, "");
+
+	err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false);
+	assert_false(err, "");
+
+	err = hpa_central_init(&test_data->central, test_data->base, hooks);
+	assert_false(err, "");
+	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
+	err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
+	    &test_data->emap, test_data->base, &test_data->shard_edata_cache,
+	    SHARD_IND, opts, sec_opts);
+	assert_false(err, "");
+
+	return (hpa_shard_t *)test_data;
+}
+
+static void
+destroy_test_data(hpa_shard_t *shard) {
+	test_data_t *test_data = (test_data_t *)shard;
+	base_delete(TSDN_NULL, test_data->base);
+	free(test_data);
+}
+
+static uintptr_t defer_bump_ptr = HUGEPAGE * 123;
+static void *
+defer_test_map(size_t size) {
+	void *result = (void *)defer_bump_ptr;
+	defer_bump_ptr += size;
+	return result;
+}
+
+static void
+defer_test_unmap(void *ptr, size_t size) {
+	(void)ptr;
+	(void)size;
+}
+
+static size_t ndefer_purge_calls = 0;
+static size_t npurge_size = 0;
+static void
+defer_test_purge(void *ptr, size_t size) {
+	(void)ptr;
+	npurge_size = size;
+	++ndefer_purge_calls;
+}
+
+static bool defer_vectorized_purge_called = false;
+static bool
+defer_vectorized_purge(void *vec, size_t vlen, size_t nbytes) {
+	(void)vec;
+	(void)nbytes;
+	++ndefer_purge_calls;
+	defer_vectorized_purge_called = true;
+	return false;
+}
+
+static size_t ndefer_hugify_calls = 0;
+static bool
+defer_test_hugify(void *ptr, size_t size, bool sync) {
+	++ndefer_hugify_calls;
+	return false;
+}
+
+static size_t ndefer_dehugify_calls = 0;
+static void
+defer_test_dehugify(void *ptr, size_t size) {
+	++ndefer_dehugify_calls;
+}
+
+static nstime_t defer_curtime;
+static void
+defer_test_curtime(nstime_t *r_time, bool first_reading) {
+	*r_time = defer_curtime;
+}
+
+static uint64_t
+defer_test_ms_since(nstime_t *past_time) {
+	return (nstime_ns(&defer_curtime) - nstime_ns(past_time)) / 1000 / 1000;
+}
+
+// test that freed pages stay in SEC and hpa thinks they are active
+
+TEST_BEGIN(test_hpa_sec) {
+	test_skip_if(!hpa_supported());
+
+	hpa_hooks_t hooks;
+	hooks.map = &defer_test_map;
+	hooks.unmap = &defer_test_unmap;
+	hooks.purge = &defer_test_purge;
+	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
+	hooks.curtime = &defer_test_curtime;
+	hooks.ms_since = &defer_test_ms_since;
+	hooks.vectorized_purge = &defer_vectorized_purge;
+
+	hpa_shard_opts_t opts = test_hpa_shard_opts;
+
+	enum { NALLOCS = 8 };
+	sec_opts_t sec_opts;
+	sec_opts.nshards = 1;
+	sec_opts.max_alloc = 2 * PAGE;
+	sec_opts.max_bytes = NALLOCS * PAGE;
+	sec_opts.batch_fill_extra = 4;
+
+	hpa_shard_t *shard = create_test_data(&hooks, &opts, &sec_opts);
+	bool         deferred_work_generated = false;
+	tsdn_t      *tsdn = tsd_tsdn(tsd_fetch());
+
+	/* alloc 1 PAGE, confirm sec has fill_extra bytes. */
+	edata_t *edata1 = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false,
+	    false, &deferred_work_generated);
+	expect_ptr_not_null(edata1, "Unexpected null edata");
+	hpa_shard_stats_t hpa_stats;
+	memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
+	hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
+	expect_zu_eq(hpa_stats.psset_stats.merged.nactive,
+	    1 + sec_opts.batch_fill_extra, "");
+	expect_zu_eq(hpa_stats.secstats.bytes, PAGE * sec_opts.batch_fill_extra,
+	    "sec should have fill extra pages");
+
+	/* Alloc/dealloc NALLOCS times and confirm extents are in sec. */
+	edata_t *edatas[NALLOCS];
+	for (int i = 0; i < NALLOCS; i++) {
+		edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false,
+		    false, false, &deferred_work_generated);
+		expect_ptr_not_null(edatas[i], "Unexpected null edata");
+	}
+	memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
+	hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
+	expect_zu_eq(hpa_stats.psset_stats.merged.nactive, 2 + NALLOCS, "");
+	expect_zu_eq(hpa_stats.secstats.bytes, PAGE, "2 refills (at 0 and 4)");
+
+	for (int i = 0; i < NALLOCS - 1; i++) {
+		pai_dalloc(
+		    tsdn, &shard->pai, edatas[i], &deferred_work_generated);
+	}
+	memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
+	hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
+	expect_zu_eq(hpa_stats.psset_stats.merged.nactive, (2 + NALLOCS), "");
+	expect_zu_eq(
+	    hpa_stats.secstats.bytes, sec_opts.max_bytes, "sec should be full");
+
+	/* this one should flush 1 + 0.25 * 8 = 3 extents */
+	pai_dalloc(
+	    tsdn, &shard->pai, edatas[NALLOCS - 1], &deferred_work_generated);
+	memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
+	hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
+	expect_zu_eq(hpa_stats.psset_stats.merged.nactive, (NALLOCS - 1), "");
+	expect_zu_eq(hpa_stats.psset_stats.merged.ndirty, 3, "");
+	expect_zu_eq(hpa_stats.secstats.bytes, 0.75 * sec_opts.max_bytes,
+	    "sec should be full");
+
+	/* Next allocation should come from SEC and not increase active */
+	edata_t *edata2 = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false,
+	    false, &deferred_work_generated);
+	expect_ptr_not_null(edata2, "Unexpected null edata");
+	memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
+	hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
+	expect_zu_eq(hpa_stats.psset_stats.merged.nactive, NALLOCS - 1, "");
+	expect_zu_eq(hpa_stats.secstats.bytes, 0.75 * sec_opts.max_bytes - PAGE,
+	    "sec should have max_bytes minus one page that just came from it");
+
+	/* We return this one and it stays in the cache */
+	pai_dalloc(tsdn, &shard->pai, edata2, &deferred_work_generated);
+	memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
+	hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
+	expect_zu_eq(hpa_stats.psset_stats.merged.nactive, NALLOCS - 1, "");
+	expect_zu_eq(hpa_stats.psset_stats.merged.ndirty, 3, "");
+	expect_zu_eq(hpa_stats.secstats.bytes, 0.75 * sec_opts.max_bytes, "");
+
+	destroy_test_data(shard);
+}
+TEST_END
+
+int
+main(void) {
+	return test_no_reentrancy(test_hpa_sec);
+}

test/unit/hpa_sec_integration.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+export MALLOC_CONF="process_madvise_max_batch:0,experimental_hpa_start_huge_if_thp_always:false"

test/unit/hpa_thp_always.c

@@ -65,10 +65,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
 
 	err = hpa_central_init(&test_data->central, test_data->base, hooks);
 	assert_false(err, "");
-
+	sec_opts_t sec_opts;
-	err = hpa_shard_init(&test_data->shard, &test_data->central,
+	sec_opts.nshards = 0;
+	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
+	err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
 	    &test_data->emap, test_data->base, &test_data->shard_edata_cache,
-	    SHARD_IND, opts);
+	    SHARD_IND, opts, &sec_opts);
 	assert_false(err, "");
 
 	return (hpa_shard_t *)test_data;

test/unit/hpa_vectorized_madvise.c

@@ -66,9 +66,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
 	err = hpa_central_init(&test_data->central, test_data->base, hooks);
 	assert_false(err, "");
 
-	err = hpa_shard_init(&test_data->shard, &test_data->central,
+	sec_opts_t sec_opts;
+	sec_opts.nshards = 0;
+	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
+	err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
 	    &test_data->emap, test_data->base, &test_data->shard_edata_cache,
-	    SHARD_IND, opts);
+	    SHARD_IND, opts, &sec_opts);
 	assert_false(err, "");
 
 	return (hpa_shard_t *)test_data;

test/unit/hpa_vectorized_madvise_large_batch.c

@@ -66,10 +66,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
 
 	err = hpa_central_init(&test_data->central, test_data->base, hooks);
 	assert_false(err, "");
-
+	sec_opts_t sec_opts;
-	err = hpa_shard_init(&test_data->shard, &test_data->central,
+	sec_opts.nshards = 0;
+	tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
+	err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
 	    &test_data->emap, test_data->base, &test_data->shard_edata_cache,
-	    SHARD_IND, opts);
+	    SHARD_IND, opts, &sec_opts);
 	assert_false(err, "");
 
 	return (hpa_shard_t *)test_data;

test/unit/mallctl.c

@@ -313,7 +313,6 @@ TEST_BEGIN(test_mallctl_opt) {
 	TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always);
 	TEST_MALLCTL_OPT(size_t, hpa_sec_max_alloc, always);
 	TEST_MALLCTL_OPT(size_t, hpa_sec_max_bytes, always);
-	TEST_MALLCTL_OPT(size_t, hpa_sec_bytes_after_flush, always);
 	TEST_MALLCTL_OPT(size_t, hpa_sec_batch_fill_extra, always);
 	TEST_MALLCTL_OPT(ssize_t, experimental_hpa_max_purge_nhp, always);
 	TEST_MALLCTL_OPT(size_t, hpa_purge_threshold, always);