Remove hpa_sec_batch_fill_extra and calculate nallocs automatically.

This change includes the following improvements:

- Remove the hpa_sec_batch_fill_extra parameter.
- Refactor the hpa_alloc() code and helper functions to be able to
  allocate more than one extent out of a single pageslab. This way
  we can amortize the per-pageslab costs (active bitmap iteration,
  pageslab metadata updates) across multiple extents.
- Decide on a min and max number of extents that will be allocated
  in hpa_alloc(). The code will try to allocate at least the min
  and allocate up to the max as long as we can allocate additional
  ones from the pageslab we already have, as additional allocations
  are relatively cheap.
- Add extent allocation distribution stats.
- Amend hpa_sec_integration.c unit test.

include/jemalloc/internal/hpa.h

@@ -50,6 +50,60 @@ struct hpa_shard_nonderived_stats_s {
 	 * Guarded by mtx.
 	 */
 	uint64_t ndehugifies;
+
+	/*
+	 * Distribution of the min number of extents we will try to allocate
+	 * from a single hpa_alloc() call.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the max number of extents we will try to allocate
+	 * from a single hpa_alloc() call.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of extents allocated for a single
+	 * hpa_alloc() call and a single mtx lock acquisition.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of ps out of which we allocated extents
+	 * for a single hpa_alloc() call and a single mtx lock acquisition.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of pages allocated from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the number of extents allocated from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
+
+	/*
+	 * Distribution of the total elapsed time (ns) for allocating extents
+	 * from a single ps.
+	 *
+	 * Guarded by mtx.
+	 */
+	uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
 };
 
 /* Completely derived; only used by CTL. */

include/jemalloc/internal/hpdata.h

@@ -432,6 +432,60 @@ void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age, bool is_huge);
 void *hpdata_reserve_alloc(hpdata_t *hpdata, size_t sz);
 void  hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz);
 
+/*
+ * For buffering extent allocations we will perform out of
+ * a single ps.
+ */
+typedef struct hpdata_alloc_offset_s hpdata_alloc_offset_t;
+struct hpdata_alloc_offset_s {
+	/*
+	 * Index on the active bitmap for the extent to allocate.
+	 * It is used to know which bits we'll need to set when we perform
+	 * the allocation. They are in the range [index, index + npages).
+	 */
+	size_t index;
+
+	/*
+	 * The length of the free bit range on the active bitmap,
+	 * starting at index, before setting the bits in the range
+	 * [index, index + npages).
+	 * It is used to determine whether one of the allocations
+	 * used up the longest free range on the active bitmap.
+	 * If it did, we might have to update the longest free range
+	 * metadata on the hpdata.
+	 */
+	size_t len_before;
+
+	/*
+	 * The length of the longest free range in the range [0, index).
+	 * When we need to update the longest free range on the hpdata,
+	 * the new value is either longest_len (the max up to index),
+	 * len_before - npages (what's left after we carve up the free
+	 * range starting at index), or the max in the range
+	 * [index + len_before, HUGEPAGE_PAGES), whichever is greater.
+	 */
+	size_t longest_len;
+};
+
+/*
+ * Given an hpdata that can serve an allocation request of size sz,
+ * find between one and max_nallocs offsets that can satisfy such
+ * an allocation request and buffer them in offsets (without actually
+ * reserving any space or updating hpdata). Return the number
+ * of offsets discovered.
+ */
+size_t hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t max_nallocs);
+/* Reserve the allocation for the given offset. */
+void *hpdata_reserve_alloc_offset(
+    hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset);
+/*
+ * Do any work that needs to be done after performing allocations
+ * from a single hpdata.
+ */
+void hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
+    hpdata_alloc_offset_t *offsets, size_t nallocs);
+
 /*
  * The hpdata_purge_prepare_t allows grabbing the metadata required to purge
  * subranges of a hugepage while holding a lock, drop the lock during the actual

include/jemalloc/internal/sec.h

@@ -97,6 +97,25 @@ sec_size_supported(sec_t *sec, size_t size) {
 	return sec_is_used(sec) && size <= sec->opts.max_alloc;
 }
 
+/* Min number of extents we will allocate when expanding the SEC. */
+#define SEC_MIN_NALLOCS 2
+
+/* Max number of extents we will allocate out of a single huge page. */
+#define SEC_MAX_NALLOCS 4
+
+/* Attempt to fill the SEC up to max_bytes / SEC_MAX_BYTES_DIV */
+#define SEC_MAX_BYTES_DIV 4
+
+/*
+ * Calculate the min and max number of extents we will try to allocate
+ * when expanding the SEC. We will attempt to allocate at least min
+ * extents and up to max extents depending on whether we can allocate
+ * them out of a huge page we have already allocated out of. Both
+ * min and max should the in the range [1, SEC_MAX_NALLOCS].
+ */
+void sec_calc_nallocs_for_size(
+    sec_t *sec, size_t size, size_t *min_nallocs, size_t *max_nallocs);
+
 /* If sec does not have extent available, it will return NULL. */
 edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
 void     sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,

include/jemalloc/internal/sec_opts.h

@@ -27,16 +27,9 @@ struct sec_opts_s {
 	 * until we are 1/4 below max_bytes.
 	 */
 	size_t max_bytes;
-	/*
-	 * When we can't satisfy an allocation out of the SEC because there are
-	 * no available ones cached, allocator will allocate a batch with extra
-	 * batch_fill_extra extents of the same size.
-	 */
-	size_t batch_fill_extra;
 };
 
 #define SEC_OPTS_NSHARDS_DEFAULT 2
-#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3
 #define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024))
 #define SEC_OPTS_MAX_BYTES_DEFAULT                                             \
 	((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT)                       \
@@ -45,6 +38,6 @@ struct sec_opts_s {
 
 #define SEC_OPTS_DEFAULT                                                       \
 	{SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT,                 \
-	    SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT}
+	    SEC_OPTS_MAX_BYTES_DEFAULT}
 
 #endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

include/jemalloc/internal/util.h

@@ -6,6 +6,18 @@
 
 #define UTIL_INLINE static inline
 
+JEMALLOC_ALWAYS_INLINE
+size_t
+min_zu(size_t a, size_t b) {
+	return (a < b) ? a : b;
+}
+
+JEMALLOC_ALWAYS_INLINE
+size_t
+max_zu(size_t a, size_t b) {
+	return (a > b) ? a : b;
+}
+
 /* Junk fill patterns. */
 #ifndef JEMALLOC_ALLOC_JUNK
 #	define JEMALLOC_ALLOC_JUNK ((uint8_t)0xa5)