diff --git a/Makefile.in b/Makefile.in index f939350f..15c295e4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -228,7 +228,9 @@ TESTS_UNIT := \ $(srcroot)test/unit/div.c \ $(srcroot)test/unit/double_free.c \ $(srcroot)test/unit/edata_cache.c \ + $(srcroot)test/unit/emap.c \ $(srcroot)test/unit/emitter.c \ + $(srcroot)test/unit/eset.c \ $(srcroot)test/unit/extent_dss.c \ $(srcroot)test/unit/extent_quantize.c \ ${srcroot}test/unit/fb.c \ @@ -238,6 +240,7 @@ TESTS_UNIT := \ ${srcroot}test/unit/san_bump.c \ $(srcroot)test/unit/hash.c \ $(srcroot)test/unit/hpa.c \ + $(srcroot)test/unit/hpa_central.c \ $(srcroot)test/unit/hpa_sec_integration.c \ $(srcroot)test/unit/hpa_thp_always.c \ $(srcroot)test/unit/hpa_vectorized_madvise.c \ @@ -266,6 +269,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/ncached_max.c \ $(srcroot)test/unit/oversize_threshold.c \ $(srcroot)test/unit/pa.c \ + $(srcroot)test/unit/pac.c \ $(srcroot)test/unit/pack.c \ $(srcroot)test/unit/pages.c \ $(srcroot)test/unit/peak.c \ @@ -305,6 +309,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/sz.c \ $(srcroot)test/unit/tcache_init.c \ $(srcroot)test/unit/tcache_max.c \ + $(srcroot)test/unit/tcache_gc.c \ $(srcroot)test/unit/test_hooks.c \ $(srcroot)test/unit/thread_event.c \ $(srcroot)test/unit/ticker.c \ diff --git a/include/jemalloc/internal/cache_bin.h b/include/jemalloc/internal/cache_bin.h index 3a7a927d..e80b6a73 100644 --- a/include/jemalloc/internal/cache_bin.h +++ b/include/jemalloc/internal/cache_bin.h @@ -523,14 +523,13 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) { /* Get the number of stashed pointers. */ JEMALLOC_ALWAYS_INLINE cache_bin_sz_t cache_bin_nstashed_get_internal(cache_bin_t *bin) { - cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin); cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get( bin); cache_bin_sz_t n = cache_bin_diff( bin, low_bits_low_bound, bin->low_bits_full) / sizeof(void *); - assert(n <= ncached_max); + assert(n <= cache_bin_ncached_max_get(bin)); if (config_debug && n != 0) { /* Below are for assertions only. */ void **low_bound = cache_bin_low_bound_get(bin); diff --git a/src/cache_bin.c b/src/cache_bin.c index ec677948..d12ce05f 100644 --- a/src/cache_bin.c +++ b/src/cache_bin.c @@ -10,8 +10,8 @@ const uintptr_t disabled_bin = JUNK_ADDR; void cache_bin_info_init(cache_bin_info_t *info, cache_bin_sz_t ncached_max) { assert(ncached_max <= CACHE_BIN_NCACHED_MAX); - size_t stack_size = (size_t)ncached_max * sizeof(void *); - assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8))); + assert((size_t)ncached_max * sizeof(void *) + < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8))); info->ncached_max = (cache_bin_sz_t)ncached_max; } @@ -94,9 +94,8 @@ cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc, bin->low_bits_full = (cache_bin_sz_t)(uintptr_t)full_position; bin->low_bits_empty = (cache_bin_sz_t)(uintptr_t)empty_position; cache_bin_info_init(&bin->bin_info, info->ncached_max); - cache_bin_sz_t free_spots = cache_bin_diff(bin, bin->low_bits_full, - (cache_bin_sz_t)(uintptr_t)bin->stack_head); - assert(free_spots == bin_stack_size); + assert(cache_bin_diff(bin, bin->low_bits_full, + (cache_bin_sz_t)(uintptr_t)bin->stack_head) == bin_stack_size); if (!cache_bin_disabled(bin)) { assert(cache_bin_ncached_get_local(bin) == 0); } diff --git a/src/tcache.c b/src/tcache.c index 8c2f6f4c..ff07c12f 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -339,6 +339,55 @@ tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote, } } +#ifdef JEMALLOC_JET +/* + * The GC helpers above are static inline so they cannot be linked from a + * separate translation unit. In JET builds we expose thin wrappers with a + * `_test` suffix so test/unit/tcache_gc.c can exercise them directly. These + * symbols are absent from the production library. + */ +#define JET_WRAP_RET(ret, fn, params, args) \ + ret fn##_test params { \ + return fn args; \ + } +#define JET_WRAP_VOID(fn, params, args) \ + void fn##_test params { \ + fn args; \ + } + +JET_WRAP_RET(cache_bin_sz_t, tcache_gc_small_nremote_get, + (cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min, + uintptr_t *addr_max, szind_t szind, size_t nflush), + (cache_bin, addr, addr_min, addr_max, szind, nflush)) + +JET_WRAP_VOID(tcache_gc_small_bin_shuffle, + (cache_bin_t *cache_bin, cache_bin_sz_t nremote, + uintptr_t addr_min, uintptr_t addr_max), + (cache_bin, nremote, addr_min, addr_max)) + +JET_WRAP_RET(uint8_t, tcache_nfill_small_lg_div_get, + (tcache_slow_t *tcache_slow, szind_t szind), + (tcache_slow, szind)) + +JET_WRAP_VOID(tcache_nfill_small_burst_prepare, + (tcache_slow_t *tcache_slow, szind_t szind), + (tcache_slow, szind)) + +JET_WRAP_VOID(tcache_nfill_small_burst_reset, + (tcache_slow_t *tcache_slow, szind_t szind), + (tcache_slow, szind)) + +JET_WRAP_VOID(tcache_nfill_small_gc_update, + (tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit), + (tcache_slow, szind, limit)) + +JET_WRAP_RET(uint8_t, tcache_gc_item_delay_compute, + (szind_t szind), (szind)) + +#undef JET_WRAP_RET +#undef JET_WRAP_VOID +#endif + static bool tcache_gc_small( tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) { diff --git a/test/unit/emap.c b/test/unit/emap.c new file mode 100644 index 00000000..14c32240 --- /dev/null +++ b/test/unit/emap.c @@ -0,0 +1,236 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/emap.h" + +#define EMAP_TEST_ARENA_IND 123 +#define EMAP_TEST_ADDR_BASE ((uintptr_t)0x40000000U) + +typedef struct emap_test_data_s emap_test_data_t; +struct emap_test_data_s { + base_t *base; + emap_t emap; + malloc_mutex_t mtx; +}; + +static edata_t * +test_edata_alloc(uintptr_t addr, size_t size, bool slab, szind_t szind, + uint64_t sn, extent_state_t state, extent_pai_t pai, + extent_head_state_t is_head) { + edata_t *edata = (edata_t *)mallocx(sizeof(edata_t), + MALLOCX_ALIGN(EDATA_ALIGNMENT)); + assert_ptr_not_null(edata, "Unexpected edata allocation failure"); + memset(edata, 0, sizeof(*edata)); + edata_init(edata, EMAP_TEST_ARENA_IND, (void *)addr, size, slab, + szind, sn, state, /* zeroed */ false, /* committed */ true, pai, + is_head); + return edata; +} + +static emap_test_data_t * +emap_test_data_create(void) { + emap_test_data_t *data = calloc(1, sizeof(*data)); + assert_ptr_not_null(data, "Unexpected calloc failure"); + data->base = base_new(TSDN_NULL, EMAP_TEST_ARENA_IND, + &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); + assert_ptr_not_null(data->base, "Unexpected base_new failure"); + assert_false(emap_init(&data->emap, data->base, /* zeroed */ true), + "Unexpected emap_init failure"); + assert_false(malloc_mutex_init(&data->mtx, "emap_test", + WITNESS_RANK_EXTENTS, malloc_mutex_rank_exclusive), + "Unexpected mutex initialization failure"); + return data; +} + +static void +emap_test_data_destroy(emap_test_data_t *data) { + base_delete(TSDN_NULL, data->base); + free(data); +} + +static void +expect_full_lookup(emap_t *emap, void *ptr, edata_t *edata, szind_t szind, + bool slab) { + emap_full_alloc_ctx_t ctx; + emap_full_alloc_ctx_lookup(TSDN_NULL, emap, ptr, &ctx); + expect_ptr_eq(edata, ctx.edata, "Unexpected emap edata"); + expect_u_eq(szind, ctx.szind, "Unexpected emap szind"); + expect_b_eq(slab, ctx.slab, "Unexpected emap slab bit"); +} + +TEST_BEGIN(test_emap_register_and_lookup_slab) { + emap_test_data_t *data = emap_test_data_create(); + + szind_t szind = 0; + edata_t *slab = test_edata_alloc(EMAP_TEST_ADDR_BASE, 4 * PAGE, + /* slab */ true, szind, 1, extent_state_active, EXTENT_PAI_PAC, + EXTENT_NOT_HEAD); + expect_false(emap_register_boundary(TSDN_NULL, &data->emap, slab, + szind, /* slab */ true), + "Unexpected boundary registration failure"); + emap_register_interior(TSDN_NULL, &data->emap, slab, szind); + expect_full_lookup(&data->emap, (void *)EMAP_TEST_ADDR_BASE, slab, + szind, true); + expect_full_lookup(&data->emap, (void *)(EMAP_TEST_ADDR_BASE + PAGE), + slab, szind, true); + expect_full_lookup(&data->emap, + (void *)(EMAP_TEST_ADDR_BASE + 3 * PAGE), slab, szind, true); + + emap_deregister_interior(TSDN_NULL, &data->emap, slab); + emap_deregister_boundary(TSDN_NULL, &data->emap, slab); + expect_ptr_null(emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)EMAP_TEST_ADDR_BASE), "Boundary should be cleared"); + + dallocx(slab, 0); + emap_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_emap_remap_updates_szind) { + emap_test_data_t *data = emap_test_data_create(); + + szind_t szind = 0; + edata_t *remap = test_edata_alloc(EMAP_TEST_ADDR_BASE + HUGEPAGE, + 2 * PAGE, /* slab */ false, SC_NSIZES, 2, extent_state_active, + EXTENT_PAI_PAC, EXTENT_NOT_HEAD); + expect_false(emap_register_boundary(TSDN_NULL, &data->emap, remap, + SC_NSIZES, /* slab */ false), + "Unexpected boundary registration failure"); + emap_remap(TSDN_NULL, &data->emap, remap, szind, /* slab */ false); + expect_full_lookup(&data->emap, edata_base_get(remap), remap, szind, + false); + expect_full_lookup(&data->emap, edata_last_get(remap), remap, + SC_NSIZES, false); + + dallocx(remap, 0); + emap_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_emap_split_then_merge) { + emap_test_data_t *data = emap_test_data_create(); + + uintptr_t split_base = EMAP_TEST_ADDR_BASE + 2 * HUGEPAGE; + edata_t *lead = test_edata_alloc(split_base, 4 * PAGE, + /* slab */ false, SC_NSIZES, 3, extent_state_active, + EXTENT_PAI_PAC, EXTENT_NOT_HEAD); + edata_t *trail = test_edata_alloc(split_base + 2 * PAGE, 2 * PAGE, + /* slab */ false, SC_NSIZES, 3, extent_state_active, + EXTENT_PAI_PAC, EXTENT_NOT_HEAD); + expect_false(emap_register_boundary(TSDN_NULL, &data->emap, lead, + SC_NSIZES, /* slab */ false), + "Unexpected boundary registration failure"); + + emap_prepare_t prepare; + expect_false(emap_split_prepare(TSDN_NULL, &data->emap, &prepare, lead, + 2 * PAGE, trail, 2 * PAGE), "Unexpected split prepare failure"); + edata_size_set(lead, 2 * PAGE); + emap_split_commit(TSDN_NULL, &data->emap, &prepare, lead, 2 * PAGE, + trail, 2 * PAGE); + expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)split_base), "Split lead base should map to lead"); + expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)(split_base + PAGE)), "Split lead end should map to lead"); + expect_ptr_eq(trail, emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)(split_base + 2 * PAGE)), + "Split trail base should map to trail"); + expect_ptr_eq(trail, emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)(split_base + 3 * PAGE)), + "Split trail end should map to trail"); + + emap_merge_prepare(TSDN_NULL, &data->emap, &prepare, lead, trail); + edata_size_set(lead, 4 * PAGE); + emap_merge_commit(TSDN_NULL, &data->emap, &prepare, lead, trail); + expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)split_base), "Merged base should map to lead"); + expect_ptr_null(emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)(split_base + PAGE)), + "Old lead boundary should be cleared after merge"); + expect_ptr_null(emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)(split_base + 2 * PAGE)), + "Old trail boundary should be cleared after merge"); + expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap, + (void *)(split_base + 3 * PAGE)), + "Merged last page should map to lead"); + + dallocx(lead, 0); + dallocx(trail, 0); + emap_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_emap_neighbor_acquisition) { + emap_test_data_t *data = emap_test_data_create(); + + edata_t *page_one = test_edata_alloc(PAGE, PAGE, /* slab */ false, + SC_NSIZES, 1, extent_state_active, EXTENT_PAI_PAC, + EXTENT_NOT_HEAD); + malloc_mutex_lock(TSDN_NULL, &data->mtx); + expect_ptr_null(emap_try_acquire_edata_neighbor(TSDN_NULL, &data->emap, + page_one, EXTENT_PAI_PAC, extent_state_dirty, + /* forward */ false), + "Backward acquisition from the first page should return NULL"); + malloc_mutex_unlock(TSDN_NULL, &data->mtx); + + uintptr_t base = EMAP_TEST_ADDR_BASE + 4 * HUGEPAGE; + edata_t *active = test_edata_alloc(base, PAGE, /* slab */ false, + SC_NSIZES, 2, extent_state_active, EXTENT_PAI_PAC, + EXTENT_NOT_HEAD); + edata_t *dirty = test_edata_alloc(base + PAGE, PAGE, + /* slab */ false, SC_NSIZES, 3, extent_state_active, + EXTENT_PAI_PAC, EXTENT_NOT_HEAD); + expect_false(emap_register_boundary(TSDN_NULL, &data->emap, active, + SC_NSIZES, /* slab */ false), "Unexpected registration failure"); + expect_false(emap_register_boundary(TSDN_NULL, &data->emap, dirty, + SC_NSIZES, /* slab */ false), "Unexpected registration failure"); + + malloc_mutex_lock(TSDN_NULL, &data->mtx); + emap_update_edata_state(TSDN_NULL, &data->emap, dirty, + extent_state_dirty); + expect_ptr_null(emap_try_acquire_edata_neighbor(TSDN_NULL, &data->emap, + active, EXTENT_PAI_PAC, extent_state_muzzy, /* forward */ true), + "State mismatch should reject neighbor acquisition"); + expect_d_eq(extent_state_dirty, edata_state_get(dirty), + "Rejected neighbor should keep its state"); + + edata_t *neighbor = emap_try_acquire_edata_neighbor(TSDN_NULL, + &data->emap, active, EXTENT_PAI_PAC, extent_state_dirty, + /* forward */ true); + expect_ptr_eq(dirty, neighbor, "Expected forward dirty neighbor"); + expect_d_eq(extent_state_merging, edata_state_get(dirty), + "Acquired neighbor should enter merging state"); + emap_release_edata(TSDN_NULL, &data->emap, dirty, extent_state_dirty); + expect_d_eq(extent_state_dirty, edata_state_get(dirty), + "Released neighbor should return to requested state"); + malloc_mutex_unlock(TSDN_NULL, &data->mtx); + + edata_t *hpa_neighbor = test_edata_alloc(base + 2 * PAGE, PAGE, + /* slab */ false, SC_NSIZES, 4, extent_state_active, + EXTENT_PAI_HPA, EXTENT_NOT_HEAD); + expect_false(emap_register_boundary(TSDN_NULL, &data->emap, + hpa_neighbor, SC_NSIZES, /* slab */ false), + "Unexpected registration failure"); + malloc_mutex_lock(TSDN_NULL, &data->mtx); + emap_update_edata_state( + TSDN_NULL, &data->emap, hpa_neighbor, extent_state_dirty); + expect_ptr_null(emap_try_acquire_edata_neighbor_expand(TSDN_NULL, + &data->emap, dirty, EXTENT_PAI_PAC, extent_state_dirty), + "PAI mismatch should reject expand acquisition"); + emap_update_edata_state(TSDN_NULL, &data->emap, hpa_neighbor, + extent_state_active); + malloc_mutex_unlock(TSDN_NULL, &data->mtx); + + dallocx(page_one, 0); + dallocx(active, 0); + dallocx(dirty, 0); + dallocx(hpa_neighbor, 0); + emap_test_data_destroy(data); +} +TEST_END + +int +main(void) { + return test_no_reentrancy(test_emap_register_and_lookup_slab, + test_emap_remap_updates_szind, + test_emap_split_then_merge, + test_emap_neighbor_acquisition); +} diff --git a/test/unit/eset.c b/test/unit/eset.c new file mode 100644 index 00000000..0422eacb --- /dev/null +++ b/test/unit/eset.c @@ -0,0 +1,176 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/eset.h" + +#define ESET_TEST_ARENA_IND 111 +#define ESET_TEST_ADDR_BASE ((uintptr_t)0x30000000U) + +static void +test_edata_init(edata_t *edata, uintptr_t addr, size_t size, uint64_t sn, + extent_state_t state, bool pinned) { + memset(edata, 0, sizeof(*edata)); + edata_init(edata, ESET_TEST_ARENA_IND, (void *)addr, size, + /* slab */ false, SC_NSIZES, sn, state, /* zeroed */ false, + /* committed */ true, EXTENT_PAI_PAC, EXTENT_NOT_HEAD); + edata_pinned_set(edata, pinned); +} + +static void +test_eset_init(eset_t *eset, extent_state_t state) { + memset(eset, 0, sizeof(*eset)); + eset_init(eset, state); +} + +TEST_BEGIN(test_eset_insert_remove_fit) { + eset_t eset; + test_eset_init(&eset, extent_state_dirty); + + edata_t a; + edata_t b; + edata_t c; + edata_t pinned; + test_edata_init(&a, ESET_TEST_ADDR_BASE, 2 * PAGE, 20, + extent_state_dirty, false); + test_edata_init(&b, ESET_TEST_ADDR_BASE + HUGEPAGE, 2 * PAGE, 10, + extent_state_dirty, false); + test_edata_init(&c, ESET_TEST_ADDR_BASE + 2 * HUGEPAGE, 4 * PAGE, 5, + extent_state_dirty, false); + test_edata_init(&pinned, ESET_TEST_ADDR_BASE + 3 * HUGEPAGE, PAGE, 1, + extent_state_dirty, true); + + eset_insert(&eset, &a); + eset_insert(&eset, &b); + eset_insert(&eset, &c); + eset_insert(&eset, &pinned); + + expect_zu_eq(9, eset_npages_get(&eset), + "Unexpected page count after inserts"); + if (config_stats) { + pszind_t pind_2p = sz_psz2ind( + sz_psz_quantize_floor(2 * PAGE)); + expect_zu_eq(2, eset_nextents_get(&eset, pind_2p), + "Unexpected extent count in 2-page bin"); + expect_zu_eq(4 * PAGE, eset_nbytes_get(&eset, pind_2p), + "Unexpected byte count in 2-page bin"); + } + + expect_ptr_eq(&a, edata_list_inactive_first(&eset.lru), + "Non-pinned extents should keep insertion LRU order"); + expect_ptr_eq(&b, edata_list_inactive_next(&eset.lru, &a), + "Non-pinned extents should keep insertion LRU order"); + expect_ptr_eq(&c, edata_list_inactive_next(&eset.lru, &b), + "Pinned extents should be excluded from the LRU"); + + edata_t *fit = eset_fit(&eset, 2 * PAGE, PAGE, + /* exact_only */ false, SC_PTR_BITS, /* prefer_small */ false); + expect_ptr_eq(&c, fit, + "Default first-fit should choose the oldest fitting extent across " + "larger bins"); + fit = eset_fit(&eset, 2 * PAGE, PAGE, + /* exact_only */ true, SC_PTR_BITS, /* prefer_small */ false); + expect_ptr_eq(&b, fit, + "Exact fit should choose the lowest serial number in the size bin"); + + eset_remove(&eset, &b); + expect_zu_eq(7, eset_npages_get(&eset), + "Unexpected page count after remove"); + fit = eset_fit(&eset, 2 * PAGE, PAGE, /* exact_only */ true, + SC_PTR_BITS, /* prefer_small */ false); + expect_ptr_eq(&a, fit, + "Removing the heap min should refresh the bin summary"); + + eset_remove(&eset, &pinned); + expect_zu_eq(6, eset_npages_get(&eset), + "Pinned removal should still update page counts"); + expect_ptr_eq(&a, edata_list_inactive_first(&eset.lru), + "Pinned removal should not disturb LRU contents"); + + eset_t prefer_eset; + test_eset_init(&prefer_eset, extent_state_dirty); + edata_t small_new; + edata_t large_old; + test_edata_init(&small_new, ESET_TEST_ADDR_BASE + 4 * HUGEPAGE, + 4 * PAGE, 100, extent_state_dirty, false); + test_edata_init(&large_old, ESET_TEST_ADDR_BASE + 5 * HUGEPAGE, + 8 * PAGE, 1, extent_state_dirty, false); + eset_insert(&prefer_eset, &small_new); + eset_insert(&prefer_eset, &large_old); + + fit = eset_fit(&prefer_eset, 3 * PAGE, PAGE, + /* exact_only */ false, SC_PTR_BITS, /* prefer_small */ false); + expect_ptr_eq(&large_old, fit, + "Default first-fit should prefer the oldest suitable extent"); + fit = eset_fit(&prefer_eset, 3 * PAGE, PAGE, + /* exact_only */ false, SC_PTR_BITS, /* prefer_small */ true); + expect_ptr_eq(&small_new, fit, + "prefer_small should stop at the smallest fitting bin"); +} +TEST_END + +TEST_BEGIN(test_eset_alignment_and_large_class_fallback) { + eset_t eset; + test_eset_init(&eset, extent_state_dirty); + + edata_t aligned_candidate; + test_edata_init(&aligned_candidate, + ESET_TEST_ADDR_BASE + 2 * PAGE, 4 * PAGE, 1, extent_state_dirty, + false); + eset_insert(&eset, &aligned_candidate); + + edata_t *fit = eset_fit(&eset, 2 * PAGE, 4 * PAGE, + /* exact_only */ false, SC_PTR_BITS, /* prefer_small */ false); + expect_ptr_eq(&aligned_candidate, fit, + "Alignment fallback should find a smaller extent that crosses the " + "requested alignment"); + + eset_t max_fit_eset; + test_eset_init(&max_fit_eset, extent_state_dirty); + edata_t too_large_old; + edata_t bounded_new; + test_edata_init(&too_large_old, ESET_TEST_ADDR_BASE + 6 * HUGEPAGE, + 64 * PAGE, 1, extent_state_dirty, false); + test_edata_init(&bounded_new, ESET_TEST_ADDR_BASE + 7 * HUGEPAGE, + 4 * PAGE, 100, extent_state_dirty, false); + eset_insert(&max_fit_eset, &too_large_old); + eset_insert(&max_fit_eset, &bounded_new); + fit = eset_fit(&max_fit_eset, 2 * PAGE, PAGE, + /* exact_only */ false, /* lg_max_fit */ 1, + /* prefer_small */ false); + expect_ptr_eq(&bounded_new, fit, + "lg_max_fit should reject excessively large older extents"); +} +TEST_END + +TEST_BEGIN(test_eset_exact_fit_large_class_disabled) { + test_skip_if(!sz_large_size_classes_disabled()); + + eset_t exact_eset; + test_eset_init(&exact_eset, extent_state_dirty); + size_t request = SC_LARGE_MINCLASS + PAGE; + edata_t exact; + edata_t larger; + test_edata_init(&exact, ESET_TEST_ADDR_BASE + 8 * HUGEPAGE, + request, 2, extent_state_dirty, false); + test_edata_init(&larger, ESET_TEST_ADDR_BASE + 9 * HUGEPAGE, + request + PAGE, 1, extent_state_dirty, false); + eset_insert(&exact_eset, &larger); + eset_insert(&exact_eset, &exact); + + edata_t *fit = eset_fit(&exact_eset, request, PAGE, + /* exact_only */ true, SC_PTR_BITS, /* prefer_small */ false); + expect_ptr_eq(&exact, fit, + "Exact search should enumerate the floor bin when large size " + "classes are disabled"); + fit = eset_fit(&exact_eset, request - PAGE, PAGE, + /* exact_only */ true, SC_PTR_BITS, /* prefer_small */ false); + expect_ptr_null(fit, + "Exact search should not return merely larger extents"); +} +TEST_END + +int +main(void) { + return test_no_reentrancy(test_eset_insert_remove_fit, + test_eset_alignment_and_large_class_fallback, + test_eset_exact_fit_large_class_disabled); +} diff --git a/test/unit/hpa_central.c b/test/unit/hpa_central.c new file mode 100644 index 00000000..e50bb1df --- /dev/null +++ b/test/unit/hpa_central.c @@ -0,0 +1,278 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/hpa.h" +#include "jemalloc/internal/hpa_central.h" + +#define HPA_TEST_ARENA_IND 125 +#define HPA_TEST_EDEN_SIZE (128 * HUGEPAGE) + +static bool hpa_test_map_fail; +static unsigned hpa_test_map_calls; +static unsigned hpa_test_unmap_calls; +static unsigned hpa_test_hugify_calls; +static void *hpa_test_last_map; +static size_t hpa_test_last_map_size; +static void *hpa_test_last_unmap; +static size_t hpa_test_last_unmap_size; +static void *hpa_test_last_hugify; +static size_t hpa_test_last_hugify_size; + +static void +hpa_test_hooks_reset(void) { + hpa_test_map_fail = false; + hpa_test_map_calls = 0; + hpa_test_unmap_calls = 0; + hpa_test_hugify_calls = 0; + hpa_test_last_map = NULL; + hpa_test_last_map_size = 0; + hpa_test_last_unmap = NULL; + hpa_test_last_unmap_size = 0; + hpa_test_last_hugify = NULL; + hpa_test_last_hugify_size = 0; +} + +static void * +hpa_test_map(size_t size) { + hpa_test_map_calls++; + hpa_test_last_map_size = size; + if (hpa_test_map_fail) { + hpa_test_last_map = NULL; + return NULL; + } + bool commit = true; + void *ret = pages_map(NULL, size, HUGEPAGE, &commit); + assert_true(commit, "HPA test mappings should be committed"); + hpa_test_last_map = ret; + return ret; +} + +static void +hpa_test_unmap(void *ptr, size_t size) { + hpa_test_unmap_calls++; + hpa_test_last_unmap = ptr; + hpa_test_last_unmap_size = size; + pages_unmap(ptr, size); +} + +static void +hpa_test_purge(void *ptr, size_t size) { +} + +static bool +hpa_test_hugify(void *ptr, size_t size, bool sync) { + hpa_test_hugify_calls++; + hpa_test_last_hugify = ptr; + hpa_test_last_hugify_size = size; + return false; +} + +static void +hpa_test_dehugify(void *ptr, size_t size) { +} + +static void +hpa_test_curtime(nstime_t *r_time, bool first_reading) { + nstime_init(r_time, 0); +} + +static uint64_t +hpa_test_ms_since(nstime_t *r_time) { + return 0; +} + +static bool +hpa_test_vectorized_purge(void *vec, size_t vlen, size_t nbytes) { + return true; +} + +static hpa_hooks_t hpa_test_hooks = { + hpa_test_map, + hpa_test_unmap, + hpa_test_purge, + hpa_test_hugify, + hpa_test_dehugify, + hpa_test_curtime, + hpa_test_ms_since, + hpa_test_vectorized_purge +}; + +static bool hpa_base_fail_after_new; +static unsigned hpa_base_alloc_calls; + +static void +hpa_base_hooks_reset(void) { + hpa_base_fail_after_new = false; + hpa_base_alloc_calls = 0; +} + +static void * +hpa_base_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { + if (hpa_base_fail_after_new && hpa_base_alloc_calls > 0) { + hpa_base_alloc_calls++; + return NULL; + } + hpa_base_alloc_calls++; + return pages_map(new_addr, size, alignment, commit); +} + +static bool +hpa_base_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, + bool committed, unsigned arena_ind) { + pages_unmap(addr, size); + return false; +} + +static void +hpa_base_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, + bool committed, unsigned arena_ind) { + pages_unmap(addr, size); +} + +static extent_hooks_t hpa_base_hooks = { + hpa_base_alloc, + hpa_base_dalloc, + hpa_base_destroy, + NULL, /* commit */ + NULL, /* decommit */ + NULL, /* purge_lazy */ + NULL, /* purge_forced */ + NULL, /* split */ + NULL /* merge */ +}; + +static hpdata_t * +hpa_central_extract_with_lock(hpa_central_t *central, malloc_mutex_t *mtx, + uint64_t age, bool hugify_eager, bool *oom) { + tsdn_t *tsdn = tsdn_fetch(); + malloc_mutex_lock(tsdn, mtx); + hpdata_t *ret = hpa_central_extract(tsdn, central, PAGE, age, + hugify_eager, oom); + malloc_mutex_unlock(tsdn, mtx); + return ret; +} + +static void +hpa_test_shard_grow_mtx_init(malloc_mutex_t *mtx) { + assert_false(malloc_mutex_init(mtx, "hpa_test_shard_grow", + WITNESS_RANK_HPA_SHARD_GROW, malloc_mutex_rank_exclusive), + "Unexpected mutex initialization failure"); +} + +TEST_BEGIN(test_hpa_central_extract_eden) { + test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit()); + hpa_test_hooks_reset(); + + tsdn_t *tsdn = tsdn_fetch(); + base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND, + &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); + assert_ptr_not_null(base, "Unexpected base_new failure"); + + hpa_central_t central; + assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), + "Unexpected hpa_central_init failure"); + malloc_mutex_t shard_grow_mtx; + hpa_test_shard_grow_mtx_init(&shard_grow_mtx); + + void *eden = NULL; + for (unsigned i = 0; i < 128; i++) { + bool oom = true; + hpdata_t *ps = hpa_central_extract_with_lock(¢ral, + &shard_grow_mtx, 1000 + i, /* hugify_eager */ true, &oom); + expect_false(oom, "Unexpected HPA central OOM"); + expect_ptr_not_null(ps, "Unexpected HPA central extraction failure"); + if (i == 0) { + eden = hpa_test_last_map; + expect_u_eq(1, hpa_test_map_calls, + "First extraction should map eden"); + expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_map_size, + "Unexpected eden mapping size"); + expect_u_eq(1, hpa_test_hugify_calls, + "Eager extraction should hugify the whole eden"); + expect_ptr_eq(eden, hpa_test_last_hugify, + "Hugify should apply to eden"); + expect_zu_eq(HPA_TEST_EDEN_SIZE, + hpa_test_last_hugify_size, + "Hugify should cover the whole eden"); + } + expect_ptr_eq((void *)((byte_t *)eden + i * HUGEPAGE), + hpdata_addr_get(ps), "Unexpected extracted pageslab addr"); + expect_u64_eq(1000 + i, hpdata_age_get(ps), + "Unexpected hpdata age"); + expect_true(hpdata_huge_get(ps), + "Eager extraction should mark hpdata huge"); + } + + expect_ptr_null(central.eden, "Exact final extraction should empty eden"); + expect_zu_eq(0, central.eden_len, + "Exact final extraction should clear eden length"); + expect_u_eq(1, hpa_test_map_calls, + "All pageslabs should come from one eden mapping"); + expect_u_eq(0, hpa_test_unmap_calls, + "Successful extraction should not unmap eden"); + + pages_unmap(eden, HPA_TEST_EDEN_SIZE); + base_delete(tsdn, base); +} +TEST_END + +TEST_BEGIN(test_hpa_central_failure_paths) { + test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit()); + hpa_test_hooks_reset(); + + tsdn_t *tsdn = tsdn_fetch(); + base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND + 1, + &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); + assert_ptr_not_null(base, "Unexpected base_new failure"); + hpa_central_t central; + assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), + "Unexpected hpa_central_init failure"); + malloc_mutex_t shard_grow_mtx; + hpa_test_shard_grow_mtx_init(&shard_grow_mtx); + + hpa_test_map_fail = true; + bool oom = false; + hpdata_t *ps = hpa_central_extract_with_lock(¢ral, + &shard_grow_mtx, 1, /* hugify_eager */ false, &oom); + expect_ptr_null(ps, "Map failure should not return hpdata"); + expect_true(oom, "Map failure should report OOM"); + expect_u_eq(1, hpa_test_map_calls, "Expected one map attempt"); + expect_u_eq(0, hpa_test_unmap_calls, + "Map failure should not call unmap"); + base_delete(tsdn, base); + + hpa_base_hooks_reset(); + hpa_test_hooks_reset(); + base = base_new(tsdn, HPA_TEST_ARENA_IND + 2, &hpa_base_hooks, + /* metadata_use_hooks */ true); + assert_ptr_not_null(base, "Unexpected base_new failure"); + hpa_base_fail_after_new = true; + while (base_alloc(tsdn, base, sizeof(hpdata_t), CACHELINE) != NULL) { + } + + assert_false(hpa_central_init(¢ral, base, &hpa_test_hooks), + "Unexpected hpa_central_init failure"); + malloc_mutex_t shard_grow_mtx2; + hpa_test_shard_grow_mtx_init(&shard_grow_mtx2); + oom = false; + ps = hpa_central_extract_with_lock(¢ral, &shard_grow_mtx2, 2, + /* hugify_eager */ false, &oom); + expect_ptr_null(ps, "Metadata OOM should not return hpdata"); + expect_true(oom, "Metadata allocation failure should report OOM"); + expect_u_eq(1, hpa_test_map_calls, + "Metadata OOM should happen after mapping eden once"); + expect_u_eq(1, hpa_test_unmap_calls, + "Metadata OOM should unmap the freshly mapped eden"); + expect_ptr_eq(hpa_test_last_map, hpa_test_last_unmap, + "Metadata OOM should unmap the eden it just mapped"); + expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_unmap_size, + "Metadata OOM should unmap the full eden"); + base_delete(tsdn, base); +} +TEST_END + +int +main(void) { + return test_no_reentrancy(test_hpa_central_extract_eden, + test_hpa_central_failure_paths); +} diff --git a/test/unit/pac.c b/test/unit/pac.c new file mode 100644 index 00000000..e1455979 --- /dev/null +++ b/test/unit/pac.c @@ -0,0 +1,395 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/pa.h" + +#define PAC_TEST_ARENA_IND 124 + +static bool pac_test_dalloc_fail; +static bool pac_test_purge_lazy_fail; +static unsigned pac_test_alloc_calls; +static unsigned pac_test_dalloc_calls; +static unsigned pac_test_destroy_calls; +static unsigned pac_test_purge_lazy_calls; + +static void +pac_test_hooks_reset(void) { + pac_test_dalloc_fail = false; + pac_test_purge_lazy_fail = false; + pac_test_alloc_calls = 0; + pac_test_dalloc_calls = 0; + pac_test_destroy_calls = 0; + pac_test_purge_lazy_calls = 0; +} + +static void * +pac_test_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, + size_t alignment, bool *zero, bool *commit, unsigned arena_ind) { + pac_test_alloc_calls++; + void *ret = pages_map(new_addr, size, alignment, commit); + return ret; +} + +static bool +pac_test_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, + bool committed, unsigned arena_ind) { + pac_test_dalloc_calls++; + if (pac_test_dalloc_fail) { + return true; + } + pages_unmap(addr, size); + return false; +} + +static void +pac_test_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, + bool committed, unsigned arena_ind) { + pac_test_destroy_calls++; + pages_unmap(addr, size); +} + +static bool +pac_test_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { + pac_test_purge_lazy_calls++; + return pac_test_purge_lazy_fail; +} + +static bool +pac_test_split(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { + return !maps_coalesce && !opt_retain; +} + +static bool +pac_test_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { + return !maps_coalesce && !opt_retain; +} + +static extent_hooks_t pac_test_hooks = { + pac_test_alloc, + pac_test_dalloc, + pac_test_destroy, + NULL, /* commit */ + NULL, /* decommit */ + pac_test_purge_lazy, + NULL, /* purge_forced */ + pac_test_split, + pac_test_merge +}; + +typedef struct pac_test_data_s pac_test_data_t; +struct pac_test_data_s { + pa_shard_t shard; + pa_central_t central; + base_t *base; + emap_t emap; + pa_shard_stats_t stats; + malloc_mutex_t stats_mtx; + extent_hooks_t hooks; +}; + +static pac_test_data_t * +pac_test_data_init(bool custom_hooks, ssize_t dirty_decay_ms, + ssize_t muzzy_decay_ms) { + tsdn_t *tsdn = tsdn_fetch(); + pac_test_data_t *data = calloc(1, sizeof(*data)); + assert_ptr_not_null(data, "Unexpected calloc failure"); + + if (custom_hooks) { + memcpy(&data->hooks, &pac_test_hooks, sizeof(extent_hooks_t)); + } else { + memcpy(&data->hooks, &ehooks_default_extent_hooks, + sizeof(extent_hooks_t)); + } + + data->base = base_new(tsdn, PAC_TEST_ARENA_IND, &data->hooks, + /* metadata_use_hooks */ true); + assert_ptr_not_null(data->base, "Unexpected base_new failure"); + assert_false(emap_init(&data->emap, data->base, /* zeroed */ true), + "Unexpected emap_init failure"); + assert_false(malloc_mutex_init(&data->stats_mtx, "pac_test_stats", + WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive), + "Unexpected stats mutex initialization failure"); + + nstime_t time; + nstime_init(&time, 0); + assert_false(pa_central_init(&data->central, data->base, /* hpa */ false, + &hpa_hooks_default), "Unexpected pa_central_init failure"); + assert_false(pa_shard_init(tsdn, &data->shard, &data->central, + &data->emap, data->base, PAC_TEST_ARENA_IND, &data->stats, + &data->stats_mtx, &time, SC_LARGE_MAXCLASS + PAGE, + dirty_decay_ms, muzzy_decay_ms), "Unexpected pa_shard_init failure"); + + return data; +} + +static void +pac_decay_all_locked(pac_t *pac, extent_state_t state, bool fully_decay) { + decay_t *decay; + pac_decay_stats_t *decay_stats; + ecache_t *ecache; + if (state == extent_state_dirty) { + decay = &pac->decay_dirty; + decay_stats = &pac->stats->decay_dirty; + ecache = &pac->ecache_dirty; + } else { + assert_d_eq(extent_state_muzzy, state, + "Only dirty and muzzy decay are supported"); + decay = &pac->decay_muzzy; + decay_stats = &pac->stats->decay_muzzy; + ecache = &pac->ecache_muzzy; + } + + tsdn_t *tsdn = tsdn_fetch(); + malloc_mutex_lock(tsdn, &decay->mtx); + pac_decay_all(tsdn, pac, decay, decay_stats, ecache, fully_decay); + malloc_mutex_unlock(tsdn, &decay->mtx); +} + +static void +pac_test_data_destroy(pac_test_data_t *data) { + /* + * Decay below calls back into the test hooks; reset all hook state + * (including the fail flags) so teardown is unaffected by anything the + * preceding test toggled. + */ + pac_test_hooks_reset(); + pac_decay_all_locked(&data->shard.pac, extent_state_dirty, + /* fully_decay */ true); + pac_decay_all_locked(&data->shard.pac, extent_state_muzzy, + /* fully_decay */ true); + pa_shard_destroy(tsdn_fetch(), &data->shard); + base_delete(tsdn_fetch(), data->base); + /* + * pac operations populated the tsd rtree_ctx with leaf-node pointers + * from the private emap we just destroyed. Invalidate the cache so + * the next test's fresh emap doesn't follow stale entries. + */ + rtree_ctx_data_init(tsd_rtree_ctx(tsd_fetch())); + free(data); +} + +static edata_t * +pac_alloc_expect(pac_test_data_t *data, size_t size, bool guarded) { + bool deferred_work_generated = false; + edata_t *edata = pac_alloc(tsdn_fetch(), &data->shard.pac, size, PAGE, + /* zero */ false, guarded, /* frequent_reuse */ false, + &deferred_work_generated); + expect_ptr_not_null(edata, "Unexpected pac_alloc failure"); + expect_zu_eq(size, edata_size_get(edata), "Unexpected allocation size"); + return edata; +} + +TEST_BEGIN(test_pac_dirty_muzzy_alloc_priority) { + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + pac_t *pac = &data->shard.pac; + size_t alloc_size = HUGEPAGE; + size_t alloc_npages = alloc_size >> LG_PAGE; + + edata_t *muzzy = pac_alloc_expect( + data, alloc_size, /* guarded */ false); + void *muzzy_addr = edata_base_get(muzzy); + edata_t *dirty = pac_alloc_expect( + data, alloc_size, /* guarded */ false); + void *dirty_addr = edata_base_get(dirty); + + bool deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, muzzy, &deferred_work_generated); + pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false); + expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_muzzy), + "Expected one muzzy page after dirty decay"); + + deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, dirty, &deferred_work_generated); + expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_dirty), + "Expected one dirty page"); + + edata_t *from_dirty = pac_alloc_expect( + data, alloc_size, /* guarded */ false); + expect_ptr_eq(dirty_addr, edata_base_get(from_dirty), + "Dirty cache should be preferred over muzzy"); + + edata_t *from_muzzy = pac_alloc_expect( + data, alloc_size, /* guarded */ false); + expect_ptr_eq(muzzy_addr, edata_base_get(from_muzzy), + "Muzzy cache should be used after dirty cache"); + + deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, from_dirty, &deferred_work_generated); + deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, from_muzzy, &deferred_work_generated); + + pac_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_pac_batched_grow_caches_trailing_dirty) { + test_skip_if(!sz_large_size_classes_disabled() + || !(maps_coalesce || opt_retain)); + + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + pac_t *pac = &data->shard.pac; + + size_t size = HUGEPAGE + PAGE; + size_t batched_size = sz_s2u_compute_using_delta(size); + size_t next_hugepage_size = HUGEPAGE_CEILING(size); + if (batched_size > next_hugepage_size) { + batched_size = next_hugepage_size; + } + assert_zu_gt(batched_size, size, + "Test size should exercise batched retained growth"); + + size_t dirty_before = ecache_npages_get(&pac->ecache_dirty); + edata_t *large = pac_alloc_expect(data, size, /* guarded */ false); + expect_zu_eq(dirty_before + ((batched_size - size) >> LG_PAGE), + ecache_npages_get(&pac->ecache_dirty), + "Batched grow should cache the trailing dirty extent"); + if (config_stats) { + expect_zu_ge(pac_mapped(pac), batched_size, + "Mapped stats should include the full batched grow"); + } + + bool deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, large, &deferred_work_generated); + + pac_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_pac_deferred_work_signals) { + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + pac_t *pac = &data->shard.pac; + + edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); + bool deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); + expect_true(deferred_work_generated, + "Non-pinned dalloc should request deferred work"); + + pac_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_pac_decay_dirty_to_muzzy_via_purge_lazy) { + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + pac_t *pac = &data->shard.pac; + + edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); + bool deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); + + unsigned purge_lazy_before = pac_test_purge_lazy_calls; + pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false); + expect_zu_eq(0, ecache_npages_get(&pac->ecache_dirty), + "Dirty decay should remove dirty pages"); + expect_zu_eq(1, ecache_npages_get(&pac->ecache_muzzy), + "Successful lazy purge should move dirty pages to muzzy"); + expect_u_gt(pac_test_purge_lazy_calls, purge_lazy_before, + "Dirty-to-muzzy decay should call purge_lazy"); + + pac_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_pac_decay_retains_when_dalloc_fails) { + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + pac_t *pac = &data->shard.pac; + + pac_test_dalloc_fail = true; + edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false); + bool deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated); + size_t retained_before = ecache_npages_get(&pac->ecache_retained); + pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ true); + expect_zu_gt(ecache_npages_get(&pac->ecache_retained), retained_before, + "Fully decayed dirty pages should be retained when dalloc fails"); + expect_u_gt(pac_test_dalloc_calls, 0, + "Fully decayed dirty pages should attempt dalloc first"); + + pac_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_pac_non_pinned_dalloc_signals_deferred_work) { + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + pac_t *pac = &data->shard.pac; + + size_t normal_size = 3 * HUGEPAGE + PAGE; + edata_t *normal = pac_alloc_expect(data, normal_size, + /* guarded */ false); + bool deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated); + expect_true(deferred_work_generated, + "Non-pinned dalloc should request deferred work"); + + pac_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_pac_non_pinned_shrink_signals_deferred_work) { + test_skip_if(!maps_coalesce); + + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + pac_t *pac = &data->shard.pac; + + size_t normal_size = 3 * HUGEPAGE + PAGE; + size_t normal_shrink_size = 3 * HUGEPAGE; + edata_t *normal = pac_alloc_expect(data, normal_size, + /* guarded */ false); + bool deferred_work_generated = false; + expect_false(pac_shrink(tsdn_fetch(), pac, normal, normal_size, + normal_shrink_size, &deferred_work_generated), + "Unexpected non-pinned shrink failure"); + expect_true(deferred_work_generated, + "Non-pinned shrink should request deferred work"); + pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated); + + pac_test_data_destroy(data); +} +TEST_END + +TEST_BEGIN(test_pac_large_guarded_dalloc_unguards_before_caching) { + pac_test_hooks_reset(); + pac_test_data_t *data = pac_test_data_init( + /* custom_hooks */ true, -1, -1); + edata_t *guarded = pac_alloc_expect(data, SC_LARGE_MINCLASS, + /* guarded */ true); + expect_true(edata_guarded_get(guarded), + "Guarded allocation should set the guarded bit"); + bool deferred_work_generated = false; + pac_dalloc(tsdn_fetch(), &data->shard.pac, guarded, + &deferred_work_generated); + expect_true(deferred_work_generated, + "Guarded dalloc should still request deferred work"); + expect_false(edata_guarded_get(guarded), + "Large guarded dalloc should unguard before caching"); + pac_test_data_destroy(data); +} +TEST_END + +int +main(void) { + return test_no_reentrancy(test_pac_dirty_muzzy_alloc_priority, + test_pac_batched_grow_caches_trailing_dirty, + test_pac_deferred_work_signals, + test_pac_decay_dirty_to_muzzy_via_purge_lazy, + test_pac_decay_retains_when_dalloc_fails, + test_pac_non_pinned_dalloc_signals_deferred_work, + test_pac_non_pinned_shrink_signals_deferred_work, + test_pac_large_guarded_dalloc_unguards_before_caching); +} diff --git a/test/unit/tcache_gc.c b/test/unit/tcache_gc.c new file mode 100644 index 00000000..7257e70a --- /dev/null +++ b/test/unit/tcache_gc.c @@ -0,0 +1,166 @@ +#include "test/jemalloc_test.h" + +extern cache_bin_sz_t tcache_gc_small_nremote_get_test( + cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min, + uintptr_t *addr_max, szind_t szind, size_t nflush); +extern void tcache_gc_small_bin_shuffle_test(cache_bin_t *cache_bin, + cache_bin_sz_t nremote, uintptr_t addr_min, uintptr_t addr_max); +extern uint8_t tcache_nfill_small_lg_div_get_test( + tcache_slow_t *tcache_slow, szind_t szind); +extern void tcache_nfill_small_burst_prepare_test( + tcache_slow_t *tcache_slow, szind_t szind); +extern void tcache_nfill_small_burst_reset_test( + tcache_slow_t *tcache_slow, szind_t szind); +extern void tcache_nfill_small_gc_update_test( + tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit); +extern uint8_t tcache_gc_item_delay_compute_test(szind_t szind); + +static void * +test_cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, + cache_bin_sz_t ncached_max) { + cache_bin_info_init(info, ncached_max); + + size_t size; + size_t alignment; + cache_bin_info_compute_alloc(info, 1, &size, &alignment); + void *mem = mallocx(size, MALLOCX_ALIGN(alignment)); + assert_ptr_not_null(mem, "Unexpected mallocx failure"); + + size_t cur_offset = 0; + cache_bin_preincrement(info, 1, mem, &cur_offset); + cache_bin_init(bin, info, mem, &cur_offset); + cache_bin_postincrement(mem, &cur_offset); + assert_zu_eq(cur_offset, size, "Should use all requested memory"); + + return mem; +} + +static void +cache_bin_fill_ptrs(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill) { + CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill); + cache_bin_init_ptr_array_for_fill(bin, &arr, nfill); + for (cache_bin_sz_t i = 0; i < nfill; i++) { + arr.ptr[i] = ptrs[i]; + } + cache_bin_finish_fill(bin, &arr, nfill); + expect_zu_eq(nfill, cache_bin_ncached_get_local(bin), + "Unexpected fill count"); +} + +TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) { + cache_bin_t bin; + cache_bin_info_t info; + void *mem = test_cache_bin_init(&bin, &info, 16); + + szind_t szind = 0; + uintptr_t anchor = ZU(0x40000000); + size_t slab_size = bin_infos[szind].slab_size; + void *ptrs[] = { + (void *)(anchor + 16), + (void *)(anchor + slab_size + 16), + (void *)(anchor + 64), + (void *)(anchor + TCACHE_GC_NEIGHBOR_LIMIT + PAGE), + }; + cache_bin_fill_ptrs(&bin, ptrs, 4); + + uintptr_t addr_min; + uintptr_t addr_max; + cache_bin_sz_t nremote = tcache_gc_small_nremote_get_test(&bin, + (void *)anchor, &addr_min, &addr_max, szind, 2); + expect_zu_eq(2, nremote, + "Should count pointers outside the local slab"); + expect_zu_eq(anchor, addr_min, "Expected slab-local lower bound"); + expect_zu_eq(anchor + slab_size, addr_max, + "Expected slab-local upper bound"); + + tcache_gc_small_bin_shuffle_test(&bin, nremote, addr_min, addr_max); + expect_ptr_eq(ptrs[0], bin.stack_head[0], + "Local pointer order should be preserved"); + expect_ptr_eq(ptrs[2], bin.stack_head[1], + "Local pointer order should be preserved"); + for (unsigned i = 2; i < 4; i++) { + expect_true((uintptr_t)bin.stack_head[i] < addr_min + || (uintptr_t)bin.stack_head[i] >= addr_max, + "Remote pointers should be moved to the flush side"); + } + + while (cache_bin_ncached_get_local(&bin) > 0) { + bool success; + cache_bin_alloc(&bin, &success); + } + cache_bin_fill_ptrs(&bin, ptrs, 4); + nremote = tcache_gc_small_nremote_get_test(&bin, (void *)anchor, + &addr_min, &addr_max, szind, 1); + expect_zu_eq(1, nremote, + "Neighbor filtering should be used when it satisfies nflush"); + expect_zu_eq(anchor - TCACHE_GC_NEIGHBOR_LIMIT, addr_min, + "Expected neighbor lower bound"); + expect_zu_eq(anchor + TCACHE_GC_NEIGHBOR_LIMIT, addr_max, + "Expected neighbor upper bound"); + + free(mem); +} +TEST_END + +TEST_BEGIN(test_tcache_gc_fill_control_and_delay) { + tcache_slow_t tcache_slow; + memset(&tcache_slow, 0, sizeof(tcache_slow)); + + szind_t szind = 0; + cache_bin_fill_ctl_t *ctl = + &tcache_slow.bin_fill_ctl_do_not_access_directly[szind]; + ctl->base = 3; + ctl->offset = 0; + + bool old_experimental_tcache_gc = opt_experimental_tcache_gc; + size_t old_tcache_gc_delay_bytes = opt_tcache_gc_delay_bytes; + + opt_experimental_tcache_gc = true; + expect_u_eq(3, tcache_nfill_small_lg_div_get_test( + &tcache_slow, szind), "Unexpected initial fill divisor"); + tcache_nfill_small_burst_prepare_test(&tcache_slow, szind); + expect_u_eq(2, tcache_nfill_small_lg_div_get_test( + &tcache_slow, szind), "Burst load should increase fill count"); + tcache_nfill_small_burst_prepare_test(&tcache_slow, szind); + expect_u_eq(1, tcache_nfill_small_lg_div_get_test( + &tcache_slow, szind), "Burst load should cap at divisor 1"); + tcache_nfill_small_burst_prepare_test(&tcache_slow, szind); + expect_u_eq(1, tcache_nfill_small_lg_div_get_test( + &tcache_slow, szind), "Burst offset should not reach base"); + + tcache_nfill_small_burst_reset_test(&tcache_slow, szind); + expect_u_eq(3, tcache_nfill_small_lg_div_get_test( + &tcache_slow, szind), "Burst reset should clear offset"); + + tcache_nfill_small_gc_update_test(&tcache_slow, szind, 0); + expect_u_eq(2, ctl->base, + "Refill during a GC period should increase future fill count"); + expect_u_eq(0, ctl->offset, "GC update should reset burst offset"); + + tcache_nfill_small_gc_update_test(&tcache_slow, szind, 64); + expect_u_eq(3, ctl->base, + "Low-water pressure should reduce future fill count"); + + ctl->offset = 2; + opt_experimental_tcache_gc = false; + expect_u_eq(3, tcache_nfill_small_lg_div_get_test( + &tcache_slow, szind), "Legacy GC should ignore burst offset"); + + size_t sz = sz_index2size(szind); + opt_tcache_gc_delay_bytes = 3 * sz; + expect_u_eq(3, tcache_gc_item_delay_compute_test(szind), + "Delay should convert bytes to items"); + opt_tcache_gc_delay_bytes = SIZE_T_MAX; + expect_u_eq(UINT8_MAX, tcache_gc_item_delay_compute_test(szind), + "Delay should saturate at uint8 max"); + + opt_experimental_tcache_gc = old_experimental_tcache_gc; + opt_tcache_gc_delay_bytes = old_tcache_gc_delay_bytes; +} +TEST_END + +int +main(void) { + return test_no_reentrancy(test_tcache_gc_small_remote_count_and_shuffle, + test_tcache_gc_fill_control_and_delay); +}