Fix psset_enumerate_search pages-vs-bytes comparison

Makefile.in

@@ -244,6 +244,7 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/hpa_vectorized_madvise.c \
 	$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \
 	$(srcroot)test/unit/hpa_background_thread.c \
+	$(srcroot)test/unit/hpa_pageslab_packing.c \
 	$(srcroot)test/unit/hpdata.c \
 	$(srcroot)test/unit/extent_alloc_flags.c \
 	$(srcroot)test/unit/huge.c \

src/psset.c

@@ -349,7 +349,7 @@ psset_enumerate_search(psset_t *psset, pszind_t pind, size_t size) {
 
 	while ((ps = hpdata_age_heap_enumerate_next(
 	            &psset->pageslabs[pind], &helper))) {
-		if (hpdata_longest_free_range_get(ps) >= size) {
+		if ((hpdata_longest_free_range_get(ps) << LG_PAGE) >= size) {
 			return ps;
 		}
 	}

test/unit/hpa_pageslab_packing.c

@@ -0,0 +1,207 @@
+#include "test/jemalloc_test.h"
+
+/*
+ * Tests that HPA cleanly packs allocations into a single pageslab when they
+ * collectively fit, instead of growing a new one.
+ */
+const char *malloc_conf =
+    "disable_large_size_classes:true,"
+    "cache_oblivious:false,"
+    "hpa_sec_nshards:0,"
+    "hpa_slab_max_alloc:2097152,"
+    "hpa_dirty_mult:-1,"
+    "hpa_hugify_delay_ms:1000000000";
+
+#define HPDATA_PAGES    (HUGEPAGE / PAGE)
+/*
+ * Every allocation must be >= SC_LARGE_MINCLASS so it goes through
+ * arena_malloc_large -> pa_alloc -> hpa_alloc with the exact size we ask
+ * for.  Anything <= SC_SMALL_MAXCLASS would route through the bin/slab
+ * path, which sizes its own slab independent of our request.
+ */
+#define LARGE_MIN_PAGES (SC_LARGE_MINCLASS / PAGE)
+
+/*
+ * Sample sizes via Fibonacci numbers, filtered at runtime to fit the current
+ * page geometry.  This spans a useful range (covers each pszind group, hits
+ * both pszind-boundary sizes like 5/8 and non-boundary sizes like
+ * 13/21/34/...) without sweeping every page count, which would balloon the
+ * 3-allocation cross product.
+ */
+static const size_t fib_pages[] = {
+	5, 8, 13, 21, 34, 55, 89, 144, 233, 377,
+};
+#define NFIB (sizeof(fib_pages) / sizeof(fib_pages[0]))
+
+static bool
+pages_testable(size_t pages) {
+	return pages >= LARGE_MIN_PAGES && pages < HPDATA_PAGES;
+}
+
+/* Shared per-test state, populated by setup_arena. */
+static int    g_alloc_flags;
+static size_t g_epoch_mib[1];
+static size_t g_epoch_miblen;
+static size_t g_npageslabs_mib[5];
+static size_t g_npageslabs_miblen;
+
+static void
+setup_arena(void) {
+	unsigned arena_ind;
+	size_t   sz = sizeof(arena_ind);
+	expect_d_eq(mallctl("arenas.create", &arena_ind, &sz, NULL, 0), 0,
+	    "arenas.create failed");
+	g_alloc_flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
+
+	g_epoch_miblen = sizeof(g_epoch_mib) / sizeof(g_epoch_mib[0]);
+	expect_d_eq(mallctlnametomib("epoch", g_epoch_mib, &g_epoch_miblen),
+	    0, "epoch mib lookup failed");
+
+	g_npageslabs_miblen =
+	    sizeof(g_npageslabs_mib) / sizeof(g_npageslabs_mib[0]);
+	expect_d_eq(mallctlnametomib("stats.arenas.0.hpa_shard.npageslabs",
+	    g_npageslabs_mib, &g_npageslabs_miblen), 0,
+	    "npageslabs mib lookup failed");
+	g_npageslabs_mib[2] = arena_ind;
+}
+
+static size_t
+get_npageslabs(void) {
+	uint64_t epoch = 1;
+	size_t   esz   = sizeof(epoch);
+	expect_d_eq(mallctlbymib(g_epoch_mib, g_epoch_miblen,
+	    &epoch, &esz, &epoch, sizeof(epoch)), 0, "epoch refresh failed");
+	size_t n;
+	size_t nsz = sizeof(n);
+	expect_d_eq(mallctlbymib(g_npageslabs_mib, g_npageslabs_miblen,
+	    &n, &nsz, NULL, 0), 0, "npageslabs read failed");
+	return n;
+}
+
+TEST_BEGIN(test_hpa_pageslab_packing_two_allocs) {
+	test_skip_if(!config_stats);
+	test_skip_if(!hpa_supported());
+	test_skip_if(opt_cache_oblivious);
+	test_skip_if(!sz_large_size_classes_disabled());
+	setup_arena();
+
+	for (size_t i = 0; i < NFIB; i++) {
+		size_t b_pages = fib_pages[i];
+		if (!pages_testable(b_pages)) {
+			continue;
+		}
+		size_t a_pages = HPDATA_PAGES - b_pages;
+		if (a_pages < LARGE_MIN_PAGES) {
+			continue;
+		}
+
+		void *a = mallocx(a_pages * PAGE, g_alloc_flags);
+		expect_ptr_not_null(a, "a mallocx(%zu PAGE) failed", a_pages);
+		void *b = mallocx(b_pages * PAGE, g_alloc_flags);
+		expect_ptr_not_null(b, "b mallocx(%zu PAGE) failed", b_pages);
+
+		expect_zu_eq(get_npageslabs(), 1,
+		    "two allocs (a=%zu b=%zu pages) should pack into one "
+		    "pageslab", a_pages, b_pages);
+
+		dallocx(b, g_alloc_flags);
+		dallocx(a, g_alloc_flags);
+	}
+}
+TEST_END
+
+TEST_BEGIN(test_hpa_pageslab_packing_three_allocs) {
+	test_skip_if(!config_stats);
+	test_skip_if(!hpa_supported());
+	test_skip_if(opt_cache_oblivious);
+	test_skip_if(!sz_large_size_classes_disabled());
+	setup_arena();
+
+	/*
+	 * Sample c (the third allocation) and a (the first allocation) from
+	 * the Fibonacci table, with a up to rest - LARGE_MIN_PAGES so that
+	 * b = rest - a is always at least LARGE_MIN_PAGES.  b is
+	 * derived to fill the rest of the pageslab.
+	 *
+	 * After packing, free the middle allocation (b) first, then a, then
+	 * c.  This exercises hpdata_unreserve's coalescing of free ranges
+	 * with both neighbors as they appear: dealloc b leaves a single
+	 * b-sized hole; dealloc a should grow that hole to a+b; dealloc c
+	 * should leave the whole pageslab empty.  As a final coalescing
+	 * check, allocate the entire pageslab in one call afterwards — that
+	 * only succeeds if the hpdata's longest_free_range was correctly
+	 * recomputed back to HPDATA_PAGES.
+	 */
+	for (size_t i = 0; i < NFIB; i++) {
+		size_t c_pages = fib_pages[i];
+		if (!pages_testable(c_pages)) {
+			continue;
+		}
+		size_t rest = HPDATA_PAGES - c_pages;
+		if (rest < 2 * LARGE_MIN_PAGES) {
+			continue;
+		}
+		for (size_t j = 0; j < NFIB; j++) {
+			size_t a_pages = fib_pages[j];
+			if (!pages_testable(a_pages)) {
+				continue;
+			}
+			if (a_pages > rest - LARGE_MIN_PAGES) {
+				continue;
+			}
+			size_t b_pages = rest - a_pages;
+			if (b_pages < LARGE_MIN_PAGES) {
+				continue;
+			}
+
+			void *a = mallocx(a_pages * PAGE, g_alloc_flags);
+			expect_ptr_not_null(a,
+			    "a mallocx(%zu PAGE) failed", a_pages);
+			void *b = mallocx(b_pages * PAGE, g_alloc_flags);
+			expect_ptr_not_null(b,
+			    "b mallocx(%zu PAGE) failed", b_pages);
+			void *c = mallocx(c_pages * PAGE, g_alloc_flags);
+			expect_ptr_not_null(c,
+			    "c mallocx(%zu PAGE) failed", c_pages);
+
+			expect_zu_eq(get_npageslabs(), 1,
+			    "three allocs (a=%zu b=%zu c=%zu pages) should "
+			    "pack into one pageslab",
+			    a_pages, b_pages, c_pages);
+
+			/* Free middle, then first, then last. */
+			dallocx(b, g_alloc_flags);
+			dallocx(a, g_alloc_flags);
+			dallocx(c, g_alloc_flags);
+
+			/*
+			 * After freeing all three the hpdata should be empty
+			 * with longest_free_range == HPDATA_PAGES.  The only
+			 * way this allocation succeeds in one pageslab is if
+			 * the coalesce on dalloc rebuilt that range.
+			 */
+			void *full = mallocx(HPDATA_PAGES * PAGE,
+			    g_alloc_flags);
+			expect_ptr_not_null(full,
+			    "full mallocx(HPDATA_PAGES) after free of "
+			    "(a=%zu b=%zu c=%zu) failed — coalesce broken?",
+			    a_pages, b_pages, c_pages);
+			expect_zu_eq(get_npageslabs(), 1,
+			    "full mallocx after free of (a=%zu b=%zu c=%zu) "
+			    "should fit the same pageslab",
+			    a_pages, b_pages, c_pages);
+			dallocx(full, g_alloc_flags);
+		}
+	}
+}
+TEST_END
+
+int
+main(void) {
+	if (config_stats && hpa_supported()) {
+		opt_hpa = true;
+	}
+	return test(
+	    test_hpa_pageslab_packing_two_allocs,
+	    test_hpa_pageslab_packing_three_allocs);
+}