diff --git a/Makefile.in b/Makefile.in index 19a98888..78546dba 100644 --- a/Makefile.in +++ b/Makefile.in @@ -244,6 +244,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/hpa_vectorized_madvise.c \ $(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \ $(srcroot)test/unit/hpa_background_thread.c \ + $(srcroot)test/unit/hpa_pageslab_packing.c \ $(srcroot)test/unit/hpdata.c \ $(srcroot)test/unit/extent_alloc_flags.c \ $(srcroot)test/unit/huge.c \ diff --git a/src/psset.c b/src/psset.c index 4e904feb..4c6ab255 100644 --- a/src/psset.c +++ b/src/psset.c @@ -349,7 +349,7 @@ psset_enumerate_search(psset_t *psset, pszind_t pind, size_t size) { while ((ps = hpdata_age_heap_enumerate_next( &psset->pageslabs[pind], &helper))) { - if (hpdata_longest_free_range_get(ps) >= size) { + if ((hpdata_longest_free_range_get(ps) << LG_PAGE) >= size) { return ps; } } diff --git a/test/unit/hpa_pageslab_packing.c b/test/unit/hpa_pageslab_packing.c new file mode 100644 index 00000000..2bd6be76 --- /dev/null +++ b/test/unit/hpa_pageslab_packing.c @@ -0,0 +1,207 @@ +#include "test/jemalloc_test.h" + +/* + * Tests that HPA cleanly packs allocations into a single pageslab when they + * collectively fit, instead of growing a new one. + */ +const char *malloc_conf = + "disable_large_size_classes:true," + "cache_oblivious:false," + "hpa_sec_nshards:0," + "hpa_slab_max_alloc:2097152," + "hpa_dirty_mult:-1," + "hpa_hugify_delay_ms:1000000000"; + +#define HPDATA_PAGES (HUGEPAGE / PAGE) +/* + * Every allocation must be >= SC_LARGE_MINCLASS so it goes through + * arena_malloc_large -> pa_alloc -> hpa_alloc with the exact size we ask + * for. Anything <= SC_SMALL_MAXCLASS would route through the bin/slab + * path, which sizes its own slab independent of our request. + */ +#define LARGE_MIN_PAGES (SC_LARGE_MINCLASS / PAGE) + +/* + * Sample sizes via Fibonacci numbers, filtered at runtime to fit the current + * page geometry. This spans a useful range (covers each pszind group, hits + * both pszind-boundary sizes like 5/8 and non-boundary sizes like + * 13/21/34/...) without sweeping every page count, which would balloon the + * 3-allocation cross product. + */ +static const size_t fib_pages[] = { + 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, +}; +#define NFIB (sizeof(fib_pages) / sizeof(fib_pages[0])) + +static bool +pages_testable(size_t pages) { + return pages >= LARGE_MIN_PAGES && pages < HPDATA_PAGES; +} + +/* Shared per-test state, populated by setup_arena. */ +static int g_alloc_flags; +static size_t g_epoch_mib[1]; +static size_t g_epoch_miblen; +static size_t g_npageslabs_mib[5]; +static size_t g_npageslabs_miblen; + +static void +setup_arena(void) { + unsigned arena_ind; + size_t sz = sizeof(arena_ind); + expect_d_eq(mallctl("arenas.create", &arena_ind, &sz, NULL, 0), 0, + "arenas.create failed"); + g_alloc_flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; + + g_epoch_miblen = sizeof(g_epoch_mib) / sizeof(g_epoch_mib[0]); + expect_d_eq(mallctlnametomib("epoch", g_epoch_mib, &g_epoch_miblen), + 0, "epoch mib lookup failed"); + + g_npageslabs_miblen = + sizeof(g_npageslabs_mib) / sizeof(g_npageslabs_mib[0]); + expect_d_eq(mallctlnametomib("stats.arenas.0.hpa_shard.npageslabs", + g_npageslabs_mib, &g_npageslabs_miblen), 0, + "npageslabs mib lookup failed"); + g_npageslabs_mib[2] = arena_ind; +} + +static size_t +get_npageslabs(void) { + uint64_t epoch = 1; + size_t esz = sizeof(epoch); + expect_d_eq(mallctlbymib(g_epoch_mib, g_epoch_miblen, + &epoch, &esz, &epoch, sizeof(epoch)), 0, "epoch refresh failed"); + size_t n; + size_t nsz = sizeof(n); + expect_d_eq(mallctlbymib(g_npageslabs_mib, g_npageslabs_miblen, + &n, &nsz, NULL, 0), 0, "npageslabs read failed"); + return n; +} + +TEST_BEGIN(test_hpa_pageslab_packing_two_allocs) { + test_skip_if(!config_stats); + test_skip_if(!hpa_supported()); + test_skip_if(opt_cache_oblivious); + test_skip_if(!sz_large_size_classes_disabled()); + setup_arena(); + + for (size_t i = 0; i < NFIB; i++) { + size_t b_pages = fib_pages[i]; + if (!pages_testable(b_pages)) { + continue; + } + size_t a_pages = HPDATA_PAGES - b_pages; + if (a_pages < LARGE_MIN_PAGES) { + continue; + } + + void *a = mallocx(a_pages * PAGE, g_alloc_flags); + expect_ptr_not_null(a, "a mallocx(%zu PAGE) failed", a_pages); + void *b = mallocx(b_pages * PAGE, g_alloc_flags); + expect_ptr_not_null(b, "b mallocx(%zu PAGE) failed", b_pages); + + expect_zu_eq(get_npageslabs(), 1, + "two allocs (a=%zu b=%zu pages) should pack into one " + "pageslab", a_pages, b_pages); + + dallocx(b, g_alloc_flags); + dallocx(a, g_alloc_flags); + } +} +TEST_END + +TEST_BEGIN(test_hpa_pageslab_packing_three_allocs) { + test_skip_if(!config_stats); + test_skip_if(!hpa_supported()); + test_skip_if(opt_cache_oblivious); + test_skip_if(!sz_large_size_classes_disabled()); + setup_arena(); + + /* + * Sample c (the third allocation) and a (the first allocation) from + * the Fibonacci table, with a up to rest - LARGE_MIN_PAGES so that + * b = rest - a is always at least LARGE_MIN_PAGES. b is + * derived to fill the rest of the pageslab. + * + * After packing, free the middle allocation (b) first, then a, then + * c. This exercises hpdata_unreserve's coalescing of free ranges + * with both neighbors as they appear: dealloc b leaves a single + * b-sized hole; dealloc a should grow that hole to a+b; dealloc c + * should leave the whole pageslab empty. As a final coalescing + * check, allocate the entire pageslab in one call afterwards — that + * only succeeds if the hpdata's longest_free_range was correctly + * recomputed back to HPDATA_PAGES. + */ + for (size_t i = 0; i < NFIB; i++) { + size_t c_pages = fib_pages[i]; + if (!pages_testable(c_pages)) { + continue; + } + size_t rest = HPDATA_PAGES - c_pages; + if (rest < 2 * LARGE_MIN_PAGES) { + continue; + } + for (size_t j = 0; j < NFIB; j++) { + size_t a_pages = fib_pages[j]; + if (!pages_testable(a_pages)) { + continue; + } + if (a_pages > rest - LARGE_MIN_PAGES) { + continue; + } + size_t b_pages = rest - a_pages; + if (b_pages < LARGE_MIN_PAGES) { + continue; + } + + void *a = mallocx(a_pages * PAGE, g_alloc_flags); + expect_ptr_not_null(a, + "a mallocx(%zu PAGE) failed", a_pages); + void *b = mallocx(b_pages * PAGE, g_alloc_flags); + expect_ptr_not_null(b, + "b mallocx(%zu PAGE) failed", b_pages); + void *c = mallocx(c_pages * PAGE, g_alloc_flags); + expect_ptr_not_null(c, + "c mallocx(%zu PAGE) failed", c_pages); + + expect_zu_eq(get_npageslabs(), 1, + "three allocs (a=%zu b=%zu c=%zu pages) should " + "pack into one pageslab", + a_pages, b_pages, c_pages); + + /* Free middle, then first, then last. */ + dallocx(b, g_alloc_flags); + dallocx(a, g_alloc_flags); + dallocx(c, g_alloc_flags); + + /* + * After freeing all three the hpdata should be empty + * with longest_free_range == HPDATA_PAGES. The only + * way this allocation succeeds in one pageslab is if + * the coalesce on dalloc rebuilt that range. + */ + void *full = mallocx(HPDATA_PAGES * PAGE, + g_alloc_flags); + expect_ptr_not_null(full, + "full mallocx(HPDATA_PAGES) after free of " + "(a=%zu b=%zu c=%zu) failed — coalesce broken?", + a_pages, b_pages, c_pages); + expect_zu_eq(get_npageslabs(), 1, + "full mallocx after free of (a=%zu b=%zu c=%zu) " + "should fit the same pageslab", + a_pages, b_pages, c_pages); + dallocx(full, g_alloc_flags); + } + } +} +TEST_END + +int +main(void) { + if (config_stats && hpa_supported()) { + opt_hpa = true; + } + return test( + test_hpa_pageslab_packing_two_allocs, + test_hpa_pageslab_packing_three_allocs); +}