jemalloc/test/unit/hpa_pageslab_packing.c

#include "test/jemalloc_test.h"

/*
 * Tests that HPA cleanly packs allocations into a single pageslab when they
 * collectively fit, instead of growing a new one.
 */
const char *malloc_conf =
    "disable_large_size_classes:true,"
    "cache_oblivious:false,"
    "hpa_sec_nshards:0,"
    "hpa_slab_max_alloc:2097152,"
    "hpa_dirty_mult:-1,"
    "hpa_hugify_delay_ms:1000000000";

#define HPDATA_PAGES    (HUGEPAGE / PAGE)
/*
 * Every allocation must be >= SC_LARGE_MINCLASS so it goes through
 * arena_malloc_large -> pa_alloc -> hpa_alloc with the exact size we ask
 * for.  Anything <= SC_SMALL_MAXCLASS would route through the bin/slab
 * path, which sizes its own slab independent of our request.
 */
#define LARGE_MIN_PAGES (SC_LARGE_MINCLASS / PAGE)

/*
 * Sample sizes via Fibonacci numbers, filtered at runtime to fit the current
 * page geometry.  This spans a useful range (covers each pszind group, hits
 * both pszind-boundary sizes like 5/8 and non-boundary sizes like
 * 13/21/34/...) without sweeping every page count, which would balloon the
 * 3-allocation cross product.
 */
static const size_t fib_pages[] = {
	5, 8, 13, 21, 34, 55, 89, 144, 233, 377,
};
#define NFIB (sizeof(fib_pages) / sizeof(fib_pages[0]))

static bool
pages_testable(size_t pages) {
	return pages >= LARGE_MIN_PAGES && pages < HPDATA_PAGES;
}

/* Shared per-test state, populated by setup_arena. */
static int    g_alloc_flags;
static size_t g_epoch_mib[1];
static size_t g_epoch_miblen;
static size_t g_npageslabs_mib[5];
static size_t g_npageslabs_miblen;

static void
setup_arena(void) {
	unsigned arena_ind;
	size_t   sz = sizeof(arena_ind);
	expect_d_eq(mallctl("arenas.create", &arena_ind, &sz, NULL, 0), 0,
	    "arenas.create failed");
	g_alloc_flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;

	g_epoch_miblen = sizeof(g_epoch_mib) / sizeof(g_epoch_mib[0]);
	expect_d_eq(mallctlnametomib("epoch", g_epoch_mib, &g_epoch_miblen),
	    0, "epoch mib lookup failed");

	g_npageslabs_miblen =
	    sizeof(g_npageslabs_mib) / sizeof(g_npageslabs_mib[0]);
	expect_d_eq(mallctlnametomib("stats.arenas.0.hpa_shard.npageslabs",
	    g_npageslabs_mib, &g_npageslabs_miblen), 0,
	    "npageslabs mib lookup failed");
	g_npageslabs_mib[2] = arena_ind;
}

static size_t
get_npageslabs(void) {
	uint64_t epoch = 1;
	size_t   esz   = sizeof(epoch);
	expect_d_eq(mallctlbymib(g_epoch_mib, g_epoch_miblen,
	    &epoch, &esz, &epoch, sizeof(epoch)), 0, "epoch refresh failed");
	size_t n;
	size_t nsz = sizeof(n);
	expect_d_eq(mallctlbymib(g_npageslabs_mib, g_npageslabs_miblen,
	    &n, &nsz, NULL, 0), 0, "npageslabs read failed");
	return n;
}

TEST_BEGIN(test_hpa_pageslab_packing_two_allocs) {
	test_skip_if(!config_stats);
	test_skip_if(!hpa_supported());
	test_skip_if(opt_cache_oblivious);
	test_skip_if(!sz_large_size_classes_disabled());
	setup_arena();

	for (size_t i = 0; i < NFIB; i++) {
		size_t b_pages = fib_pages[i];
		if (!pages_testable(b_pages)) {
			continue;
		}
		size_t a_pages = HPDATA_PAGES - b_pages;
		if (a_pages < LARGE_MIN_PAGES) {
			continue;
		}

		void *a = mallocx(a_pages * PAGE, g_alloc_flags);
		expect_ptr_not_null(a, "a mallocx(%zu PAGE) failed", a_pages);
		void *b = mallocx(b_pages * PAGE, g_alloc_flags);
		expect_ptr_not_null(b, "b mallocx(%zu PAGE) failed", b_pages);

		expect_zu_eq(get_npageslabs(), 1,
		    "two allocs (a=%zu b=%zu pages) should pack into one "
		    "pageslab", a_pages, b_pages);

		dallocx(b, g_alloc_flags);
		dallocx(a, g_alloc_flags);
	}
}
TEST_END

TEST_BEGIN(test_hpa_pageslab_packing_three_allocs) {
	test_skip_if(!config_stats);
	test_skip_if(!hpa_supported());
	test_skip_if(opt_cache_oblivious);
	test_skip_if(!sz_large_size_classes_disabled());
	setup_arena();

	/*
	 * Sample c (the third allocation) and a (the first allocation) from
	 * the Fibonacci table, with a up to rest - LARGE_MIN_PAGES so that
	 * b = rest - a is always at least LARGE_MIN_PAGES.  b is
	 * derived to fill the rest of the pageslab.
	 *
	 * After packing, free the middle allocation (b) first, then a, then
	 * c.  This exercises hpdata_unreserve's coalescing of free ranges
	 * with both neighbors as they appear: dealloc b leaves a single
	 * b-sized hole; dealloc a should grow that hole to a+b; dealloc c
	 * should leave the whole pageslab empty.  As a final coalescing
	 * check, allocate the entire pageslab in one call afterwards — that
	 * only succeeds if the hpdata's longest_free_range was correctly
	 * recomputed back to HPDATA_PAGES.
	 */
	for (size_t i = 0; i < NFIB; i++) {
		size_t c_pages = fib_pages[i];
		if (!pages_testable(c_pages)) {
			continue;
		}
		size_t rest = HPDATA_PAGES - c_pages;
		if (rest < 2 * LARGE_MIN_PAGES) {
			continue;
		}
		for (size_t j = 0; j < NFIB; j++) {
			size_t a_pages = fib_pages[j];
			if (!pages_testable(a_pages)) {
				continue;
			}
			if (a_pages > rest - LARGE_MIN_PAGES) {
				continue;
			}
			size_t b_pages = rest - a_pages;
			if (b_pages < LARGE_MIN_PAGES) {
				continue;
			}

			void *a = mallocx(a_pages * PAGE, g_alloc_flags);
			expect_ptr_not_null(a,
			    "a mallocx(%zu PAGE) failed", a_pages);
			void *b = mallocx(b_pages * PAGE, g_alloc_flags);
			expect_ptr_not_null(b,
			    "b mallocx(%zu PAGE) failed", b_pages);
			void *c = mallocx(c_pages * PAGE, g_alloc_flags);
			expect_ptr_not_null(c,
			    "c mallocx(%zu PAGE) failed", c_pages);

			expect_zu_eq(get_npageslabs(), 1,
			    "three allocs (a=%zu b=%zu c=%zu pages) should "
			    "pack into one pageslab",
			    a_pages, b_pages, c_pages);

			/* Free middle, then first, then last. */
			dallocx(b, g_alloc_flags);
			dallocx(a, g_alloc_flags);
			dallocx(c, g_alloc_flags);

			/*
			 * After freeing all three the hpdata should be empty
			 * with longest_free_range == HPDATA_PAGES.  The only
			 * way this allocation succeeds in one pageslab is if
			 * the coalesce on dalloc rebuilt that range.
			 */
			void *full = mallocx(HPDATA_PAGES * PAGE,
			    g_alloc_flags);
			expect_ptr_not_null(full,
			    "full mallocx(HPDATA_PAGES) after free of "
			    "(a=%zu b=%zu c=%zu) failed — coalesce broken?",
			    a_pages, b_pages, c_pages);
			expect_zu_eq(get_npageslabs(), 1,
			    "full mallocx after free of (a=%zu b=%zu c=%zu) "
			    "should fit the same pageslab",
			    a_pages, b_pages, c_pages);
			dallocx(full, g_alloc_flags);
		}
	}
}
TEST_END

int
main(void) {
	if (config_stats && hpa_supported()) {
		opt_hpa = true;
	}
	return test(
	    test_hpa_pageslab_packing_two_allocs,
	    test_hpa_pageslab_packing_three_allocs);
}