romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-06-01 17:54:16 +03:00
Code Issues Projects Releases Packages Wiki Activity

Merge 235be45153 into 6b24522545

Browse source
This commit is contained in:
Slobodan Predolac 2026-05-27 21:49:01 +00:00 • committed by GitHub
commit 208d067394
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 1310 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

5
Makefile.in
View file

@ -228,7 +228,9 @@ TESTS_UNIT := \
$(srcroot)test/unit/div.c \
$(srcroot)test/unit/double_free.c \
$(srcroot)test/unit/edata_cache.c \
$(srcroot)test/unit/emap.c \
$(srcroot)test/unit/emitter.c \
$(srcroot)test/unit/eset.c \
$(srcroot)test/unit/extent_dss.c \
$(srcroot)test/unit/extent_quantize.c \
${srcroot}test/unit/fb.c \
@ -238,6 +240,7 @@ TESTS_UNIT := \
${srcroot}test/unit/san_bump.c \
$(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hpa.c \
$(srcroot)test/unit/hpa_central.c \
$(srcroot)test/unit/hpa_sec_integration.c \
$(srcroot)test/unit/hpa_thp_always.c \
$(srcroot)test/unit/hpa_vectorized_madvise.c \
@ -266,6 +269,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/ncached_max.c \
$(srcroot)test/unit/oversize_threshold.c \
$(srcroot)test/unit/pa.c \
$(srcroot)test/unit/pac.c \
$(srcroot)test/unit/pack.c \
$(srcroot)test/unit/pages.c \
$(srcroot)test/unit/peak.c \
@ -305,6 +309,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/sz.c \
$(srcroot)test/unit/tcache_init.c \
$(srcroot)test/unit/tcache_max.c \
$(srcroot)test/unit/tcache_gc.c \
$(srcroot)test/unit/test_hooks.c \
$(srcroot)test/unit/thread_event.c \
$(srcroot)test/unit/ticker.c \

3
include/jemalloc/internal/cache_bin.h
View file

@ -523,14 +523,13 @@ cache_bin_stash(cache_bin_t *bin, void *ptr) {
/* Get the number of stashed pointers. */
JEMALLOC_ALWAYS_INLINE cache_bin_sz_t
cache_bin_nstashed_get_internal(cache_bin_t *bin) {
cache_bin_sz_t ncached_max = cache_bin_ncached_max_get(bin);
cache_bin_sz_t low_bits_low_bound = cache_bin_low_bits_low_bound_get(
bin);
cache_bin_sz_t n = cache_bin_diff(
bin, low_bits_low_bound, bin->low_bits_full)
/ sizeof(void *);
assert(n <= ncached_max);
assert(n <= cache_bin_ncached_max_get(bin));
if (config_debug && n != 0) {
/* Below are for assertions only. */
void **low_bound = cache_bin_low_bound_get(bin);

9
src/cache_bin.c
View file

@ -10,8 +10,8 @@ const uintptr_t disabled_bin = JUNK_ADDR;
void
cache_bin_info_init(cache_bin_info_t *info, cache_bin_sz_t ncached_max) {
assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
size_t stack_size = (size_t)ncached_max * sizeof(void *);
assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
assert((size_t)ncached_max * sizeof(void *)
< ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
info->ncached_max = (cache_bin_sz_t)ncached_max;
}
@ -94,9 +94,8 @@ cache_bin_init(cache_bin_t *bin, const cache_bin_info_t *info, void *alloc,
bin->low_bits_full = (cache_bin_sz_t)(uintptr_t)full_position;
bin->low_bits_empty = (cache_bin_sz_t)(uintptr_t)empty_position;
cache_bin_info_init(&bin->bin_info, info->ncached_max);
cache_bin_sz_t free_spots = cache_bin_diff(bin, bin->low_bits_full,
(cache_bin_sz_t)(uintptr_t)bin->stack_head);
assert(free_spots == bin_stack_size);
assert(cache_bin_diff(bin, bin->low_bits_full,
(cache_bin_sz_t)(uintptr_t)bin->stack_head) == bin_stack_size);
if (!cache_bin_disabled(bin)) {
assert(cache_bin_ncached_get_local(bin) == 0);
}

49
src/tcache.c
View file

@ -339,6 +339,55 @@ tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
}
}
#ifdef JEMALLOC_JET
/*
* The GC helpers above are static inline so they cannot be linked from a
* separate translation unit. In JET builds we expose thin wrappers with a
* `_test` suffix so test/unit/tcache_gc.c can exercise them directly. These
* symbols are absent from the production library.
*/
#define JET_WRAP_RET(ret, fn, params, args) \
ret fn##_test params { \
return fn args; \
}
#define JET_WRAP_VOID(fn, params, args) \
void fn##_test params { \
fn args; \
}
JET_WRAP_RET(cache_bin_sz_t, tcache_gc_small_nremote_get,
(cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min,
uintptr_t *addr_max, szind_t szind, size_t nflush),
(cache_bin, addr, addr_min, addr_max, szind, nflush))
JET_WRAP_VOID(tcache_gc_small_bin_shuffle,
(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
uintptr_t addr_min, uintptr_t addr_max),
(cache_bin, nremote, addr_min, addr_max))
JET_WRAP_RET(uint8_t, tcache_nfill_small_lg_div_get,
(tcache_slow_t *tcache_slow, szind_t szind),
(tcache_slow, szind))
JET_WRAP_VOID(tcache_nfill_small_burst_prepare,
(tcache_slow_t *tcache_slow, szind_t szind),
(tcache_slow, szind))
JET_WRAP_VOID(tcache_nfill_small_burst_reset,
(tcache_slow_t *tcache_slow, szind_t szind),
(tcache_slow, szind))
JET_WRAP_VOID(tcache_nfill_small_gc_update,
(tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit),
(tcache_slow, szind, limit))
JET_WRAP_RET(uint8_t, tcache_gc_item_delay_compute,
(szind_t szind), (szind))
#undef JET_WRAP_RET
#undef JET_WRAP_VOID
#endif
static bool
tcache_gc_small(
tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) {

236
test/unit/emap.c Normal file
View file

@ -0,0 +1,236 @@
#include "test/jemalloc_test.h"
#include "jemalloc/internal/emap.h"
#define EMAP_TEST_ARENA_IND 123
#define EMAP_TEST_ADDR_BASE ((uintptr_t)0x40000000U)
typedef struct emap_test_data_s emap_test_data_t;
struct emap_test_data_s {
base_t *base;
emap_t emap;
malloc_mutex_t mtx;
};
static edata_t *
test_edata_alloc(uintptr_t addr, size_t size, bool slab, szind_t szind,
uint64_t sn, extent_state_t state, extent_pai_t pai,
extent_head_state_t is_head) {
edata_t *edata = (edata_t *)mallocx(sizeof(edata_t),
MALLOCX_ALIGN(EDATA_ALIGNMENT));
assert_ptr_not_null(edata, "Unexpected edata allocation failure");
memset(edata, 0, sizeof(*edata));
edata_init(edata, EMAP_TEST_ARENA_IND, (void *)addr, size, slab,
szind, sn, state, /* zeroed */ false, /* committed */ true, pai,
is_head);
return edata;
}
static emap_test_data_t *
emap_test_data_create(void) {
emap_test_data_t *data = calloc(1, sizeof(*data));
assert_ptr_not_null(data, "Unexpected calloc failure");
data->base = base_new(TSDN_NULL, EMAP_TEST_ARENA_IND,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(data->base, "Unexpected base_new failure");
assert_false(emap_init(&data->emap, data->base, /* zeroed */ true),
"Unexpected emap_init failure");
assert_false(malloc_mutex_init(&data->mtx, "emap_test",
WITNESS_RANK_EXTENTS, malloc_mutex_rank_exclusive),
"Unexpected mutex initialization failure");
return data;
}
static void
emap_test_data_destroy(emap_test_data_t *data) {
base_delete(TSDN_NULL, data->base);
free(data);
}
static void
expect_full_lookup(emap_t *emap, void *ptr, edata_t *edata, szind_t szind,
bool slab) {
emap_full_alloc_ctx_t ctx;
emap_full_alloc_ctx_lookup(TSDN_NULL, emap, ptr, &ctx);
expect_ptr_eq(edata, ctx.edata, "Unexpected emap edata");
expect_u_eq(szind, ctx.szind, "Unexpected emap szind");
expect_b_eq(slab, ctx.slab, "Unexpected emap slab bit");
}
TEST_BEGIN(test_emap_register_and_lookup_slab) {
emap_test_data_t *data = emap_test_data_create();
szind_t szind = 0;
edata_t *slab = test_edata_alloc(EMAP_TEST_ADDR_BASE, 4 * PAGE,
/* slab */ true, szind, 1, extent_state_active, EXTENT_PAI_PAC,
EXTENT_NOT_HEAD);
expect_false(emap_register_boundary(TSDN_NULL, &data->emap, slab,
szind, /* slab */ true),
"Unexpected boundary registration failure");
emap_register_interior(TSDN_NULL, &data->emap, slab, szind);
expect_full_lookup(&data->emap, (void *)EMAP_TEST_ADDR_BASE, slab,
szind, true);
expect_full_lookup(&data->emap, (void *)(EMAP_TEST_ADDR_BASE + PAGE),
slab, szind, true);
expect_full_lookup(&data->emap,
(void *)(EMAP_TEST_ADDR_BASE + 3 * PAGE), slab, szind, true);
emap_deregister_interior(TSDN_NULL, &data->emap, slab);
emap_deregister_boundary(TSDN_NULL, &data->emap, slab);
expect_ptr_null(emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)EMAP_TEST_ADDR_BASE), "Boundary should be cleared");
dallocx(slab, 0);
emap_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_emap_remap_updates_szind) {
emap_test_data_t *data = emap_test_data_create();
szind_t szind = 0;
edata_t *remap = test_edata_alloc(EMAP_TEST_ADDR_BASE + HUGEPAGE,
2 * PAGE, /* slab */ false, SC_NSIZES, 2, extent_state_active,
EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
expect_false(emap_register_boundary(TSDN_NULL, &data->emap, remap,
SC_NSIZES, /* slab */ false),
"Unexpected boundary registration failure");
emap_remap(TSDN_NULL, &data->emap, remap, szind, /* slab */ false);
expect_full_lookup(&data->emap, edata_base_get(remap), remap, szind,
false);
expect_full_lookup(&data->emap, edata_last_get(remap), remap,
SC_NSIZES, false);
dallocx(remap, 0);
emap_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_emap_split_then_merge) {
emap_test_data_t *data = emap_test_data_create();
uintptr_t split_base = EMAP_TEST_ADDR_BASE + 2 * HUGEPAGE;
edata_t *lead = test_edata_alloc(split_base, 4 * PAGE,
/* slab */ false, SC_NSIZES, 3, extent_state_active,
EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
edata_t *trail = test_edata_alloc(split_base + 2 * PAGE, 2 * PAGE,
/* slab */ false, SC_NSIZES, 3, extent_state_active,
EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
expect_false(emap_register_boundary(TSDN_NULL, &data->emap, lead,
SC_NSIZES, /* slab */ false),
"Unexpected boundary registration failure");
emap_prepare_t prepare;
expect_false(emap_split_prepare(TSDN_NULL, &data->emap, &prepare, lead,
2 * PAGE, trail, 2 * PAGE), "Unexpected split prepare failure");
edata_size_set(lead, 2 * PAGE);
emap_split_commit(TSDN_NULL, &data->emap, &prepare, lead, 2 * PAGE,
trail, 2 * PAGE);
expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)split_base), "Split lead base should map to lead");
expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)(split_base + PAGE)), "Split lead end should map to lead");
expect_ptr_eq(trail, emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)(split_base + 2 * PAGE)),
"Split trail base should map to trail");
expect_ptr_eq(trail, emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)(split_base + 3 * PAGE)),
"Split trail end should map to trail");
emap_merge_prepare(TSDN_NULL, &data->emap, &prepare, lead, trail);
edata_size_set(lead, 4 * PAGE);
emap_merge_commit(TSDN_NULL, &data->emap, &prepare, lead, trail);
expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)split_base), "Merged base should map to lead");
expect_ptr_null(emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)(split_base + PAGE)),
"Old lead boundary should be cleared after merge");
expect_ptr_null(emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)(split_base + 2 * PAGE)),
"Old trail boundary should be cleared after merge");
expect_ptr_eq(lead, emap_edata_lookup(TSDN_NULL, &data->emap,
(void *)(split_base + 3 * PAGE)),
"Merged last page should map to lead");
dallocx(lead, 0);
dallocx(trail, 0);
emap_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_emap_neighbor_acquisition) {
emap_test_data_t *data = emap_test_data_create();
edata_t *page_one = test_edata_alloc(PAGE, PAGE, /* slab */ false,
SC_NSIZES, 1, extent_state_active, EXTENT_PAI_PAC,
EXTENT_NOT_HEAD);
malloc_mutex_lock(TSDN_NULL, &data->mtx);
expect_ptr_null(emap_try_acquire_edata_neighbor(TSDN_NULL, &data->emap,
page_one, EXTENT_PAI_PAC, extent_state_dirty,
/* forward */ false),
"Backward acquisition from the first page should return NULL");
malloc_mutex_unlock(TSDN_NULL, &data->mtx);
uintptr_t base = EMAP_TEST_ADDR_BASE + 4 * HUGEPAGE;
edata_t *active = test_edata_alloc(base, PAGE, /* slab */ false,
SC_NSIZES, 2, extent_state_active, EXTENT_PAI_PAC,
EXTENT_NOT_HEAD);
edata_t *dirty = test_edata_alloc(base + PAGE, PAGE,
/* slab */ false, SC_NSIZES, 3, extent_state_active,
EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
expect_false(emap_register_boundary(TSDN_NULL, &data->emap, active,
SC_NSIZES, /* slab */ false), "Unexpected registration failure");
expect_false(emap_register_boundary(TSDN_NULL, &data->emap, dirty,
SC_NSIZES, /* slab */ false), "Unexpected registration failure");
malloc_mutex_lock(TSDN_NULL, &data->mtx);
emap_update_edata_state(TSDN_NULL, &data->emap, dirty,
extent_state_dirty);
expect_ptr_null(emap_try_acquire_edata_neighbor(TSDN_NULL, &data->emap,
active, EXTENT_PAI_PAC, extent_state_muzzy, /* forward */ true),
"State mismatch should reject neighbor acquisition");
expect_d_eq(extent_state_dirty, edata_state_get(dirty),
"Rejected neighbor should keep its state");
edata_t *neighbor = emap_try_acquire_edata_neighbor(TSDN_NULL,
&data->emap, active, EXTENT_PAI_PAC, extent_state_dirty,
/* forward */ true);
expect_ptr_eq(dirty, neighbor, "Expected forward dirty neighbor");
expect_d_eq(extent_state_merging, edata_state_get(dirty),
"Acquired neighbor should enter merging state");
emap_release_edata(TSDN_NULL, &data->emap, dirty, extent_state_dirty);
expect_d_eq(extent_state_dirty, edata_state_get(dirty),
"Released neighbor should return to requested state");
malloc_mutex_unlock(TSDN_NULL, &data->mtx);
edata_t *hpa_neighbor = test_edata_alloc(base + 2 * PAGE, PAGE,
/* slab */ false, SC_NSIZES, 4, extent_state_active,
EXTENT_PAI_HPA, EXTENT_NOT_HEAD);
expect_false(emap_register_boundary(TSDN_NULL, &data->emap,
hpa_neighbor, SC_NSIZES, /* slab */ false),
"Unexpected registration failure");
malloc_mutex_lock(TSDN_NULL, &data->mtx);
emap_update_edata_state(
TSDN_NULL, &data->emap, hpa_neighbor, extent_state_dirty);
expect_ptr_null(emap_try_acquire_edata_neighbor_expand(TSDN_NULL,
&data->emap, dirty, EXTENT_PAI_PAC, extent_state_dirty),
"PAI mismatch should reject expand acquisition");
emap_update_edata_state(TSDN_NULL, &data->emap, hpa_neighbor,
extent_state_active);
malloc_mutex_unlock(TSDN_NULL, &data->mtx);
dallocx(page_one, 0);
dallocx(active, 0);
dallocx(dirty, 0);
dallocx(hpa_neighbor, 0);
emap_test_data_destroy(data);
}
TEST_END
int
main(void) {
return test_no_reentrancy(test_emap_register_and_lookup_slab,
test_emap_remap_updates_szind,
test_emap_split_then_merge,
test_emap_neighbor_acquisition);
}

176
test/unit/eset.c Normal file
View file

@ -0,0 +1,176 @@
#include "test/jemalloc_test.h"
#include "jemalloc/internal/eset.h"
#define ESET_TEST_ARENA_IND 111
#define ESET_TEST_ADDR_BASE ((uintptr_t)0x30000000U)
static void
test_edata_init(edata_t *edata, uintptr_t addr, size_t size, uint64_t sn,
extent_state_t state, bool pinned) {
memset(edata, 0, sizeof(*edata));
edata_init(edata, ESET_TEST_ARENA_IND, (void *)addr, size,
/* slab */ false, SC_NSIZES, sn, state, /* zeroed */ false,
/* committed */ true, EXTENT_PAI_PAC, EXTENT_NOT_HEAD);
edata_pinned_set(edata, pinned);
}
static void
test_eset_init(eset_t *eset, extent_state_t state) {
memset(eset, 0, sizeof(*eset));
eset_init(eset, state);
}
TEST_BEGIN(test_eset_insert_remove_fit) {
eset_t eset;
test_eset_init(&eset, extent_state_dirty);
edata_t a;
edata_t b;
edata_t c;
edata_t pinned;
test_edata_init(&a, ESET_TEST_ADDR_BASE, 2 * PAGE, 20,
extent_state_dirty, false);
test_edata_init(&b, ESET_TEST_ADDR_BASE + HUGEPAGE, 2 * PAGE, 10,
extent_state_dirty, false);
test_edata_init(&c, ESET_TEST_ADDR_BASE + 2 * HUGEPAGE, 4 * PAGE, 5,
extent_state_dirty, false);
test_edata_init(&pinned, ESET_TEST_ADDR_BASE + 3 * HUGEPAGE, PAGE, 1,
extent_state_dirty, true);
eset_insert(&eset, &a);
eset_insert(&eset, &b);
eset_insert(&eset, &c);
eset_insert(&eset, &pinned);
expect_zu_eq(9, eset_npages_get(&eset),
"Unexpected page count after inserts");
if (config_stats) {
pszind_t pind_2p = sz_psz2ind(
sz_psz_quantize_floor(2 * PAGE));
expect_zu_eq(2, eset_nextents_get(&eset, pind_2p),
"Unexpected extent count in 2-page bin");
expect_zu_eq(4 * PAGE, eset_nbytes_get(&eset, pind_2p),
"Unexpected byte count in 2-page bin");
}
expect_ptr_eq(&a, edata_list_inactive_first(&eset.lru),
"Non-pinned extents should keep insertion LRU order");
expect_ptr_eq(&b, edata_list_inactive_next(&eset.lru, &a),
"Non-pinned extents should keep insertion LRU order");
expect_ptr_eq(&c, edata_list_inactive_next(&eset.lru, &b),
"Pinned extents should be excluded from the LRU");
edata_t *fit = eset_fit(&eset, 2 * PAGE, PAGE,
/* exact_only */ false, SC_PTR_BITS, /* prefer_small */ false);
expect_ptr_eq(&c, fit,
"Default first-fit should choose the oldest fitting extent across "
"larger bins");
fit = eset_fit(&eset, 2 * PAGE, PAGE,
/* exact_only */ true, SC_PTR_BITS, /* prefer_small */ false);
expect_ptr_eq(&b, fit,
"Exact fit should choose the lowest serial number in the size bin");
eset_remove(&eset, &b);
expect_zu_eq(7, eset_npages_get(&eset),
"Unexpected page count after remove");
fit = eset_fit(&eset, 2 * PAGE, PAGE, /* exact_only */ true,
SC_PTR_BITS, /* prefer_small */ false);
expect_ptr_eq(&a, fit,
"Removing the heap min should refresh the bin summary");
eset_remove(&eset, &pinned);
expect_zu_eq(6, eset_npages_get(&eset),
"Pinned removal should still update page counts");
expect_ptr_eq(&a, edata_list_inactive_first(&eset.lru),
"Pinned removal should not disturb LRU contents");
eset_t prefer_eset;
test_eset_init(&prefer_eset, extent_state_dirty);
edata_t small_new;
edata_t large_old;
test_edata_init(&small_new, ESET_TEST_ADDR_BASE + 4 * HUGEPAGE,
4 * PAGE, 100, extent_state_dirty, false);
test_edata_init(&large_old, ESET_TEST_ADDR_BASE + 5 * HUGEPAGE,
8 * PAGE, 1, extent_state_dirty, false);
eset_insert(&prefer_eset, &small_new);
eset_insert(&prefer_eset, &large_old);
fit = eset_fit(&prefer_eset, 3 * PAGE, PAGE,
/* exact_only */ false, SC_PTR_BITS, /* prefer_small */ false);
expect_ptr_eq(&large_old, fit,
"Default first-fit should prefer the oldest suitable extent");
fit = eset_fit(&prefer_eset, 3 * PAGE, PAGE,
/* exact_only */ false, SC_PTR_BITS, /* prefer_small */ true);
expect_ptr_eq(&small_new, fit,
"prefer_small should stop at the smallest fitting bin");
}
TEST_END
TEST_BEGIN(test_eset_alignment_and_large_class_fallback) {
eset_t eset;
test_eset_init(&eset, extent_state_dirty);
edata_t aligned_candidate;
test_edata_init(&aligned_candidate,
ESET_TEST_ADDR_BASE + 2 * PAGE, 4 * PAGE, 1, extent_state_dirty,
false);
eset_insert(&eset, &aligned_candidate);
edata_t *fit = eset_fit(&eset, 2 * PAGE, 4 * PAGE,
/* exact_only */ false, SC_PTR_BITS, /* prefer_small */ false);
expect_ptr_eq(&aligned_candidate, fit,
"Alignment fallback should find a smaller extent that crosses the "
"requested alignment");
eset_t max_fit_eset;
test_eset_init(&max_fit_eset, extent_state_dirty);
edata_t too_large_old;
edata_t bounded_new;
test_edata_init(&too_large_old, ESET_TEST_ADDR_BASE + 6 * HUGEPAGE,
64 * PAGE, 1, extent_state_dirty, false);
test_edata_init(&bounded_new, ESET_TEST_ADDR_BASE + 7 * HUGEPAGE,
4 * PAGE, 100, extent_state_dirty, false);
eset_insert(&max_fit_eset, &too_large_old);
eset_insert(&max_fit_eset, &bounded_new);
fit = eset_fit(&max_fit_eset, 2 * PAGE, PAGE,
/* exact_only */ false, /* lg_max_fit */ 1,
/* prefer_small */ false);
expect_ptr_eq(&bounded_new, fit,
"lg_max_fit should reject excessively large older extents");
}
TEST_END
TEST_BEGIN(test_eset_exact_fit_large_class_disabled) {
test_skip_if(!sz_large_size_classes_disabled());
eset_t exact_eset;
test_eset_init(&exact_eset, extent_state_dirty);
size_t request = SC_LARGE_MINCLASS + PAGE;
edata_t exact;
edata_t larger;
test_edata_init(&exact, ESET_TEST_ADDR_BASE + 8 * HUGEPAGE,
request, 2, extent_state_dirty, false);
test_edata_init(&larger, ESET_TEST_ADDR_BASE + 9 * HUGEPAGE,
request + PAGE, 1, extent_state_dirty, false);
eset_insert(&exact_eset, &larger);
eset_insert(&exact_eset, &exact);
edata_t *fit = eset_fit(&exact_eset, request, PAGE,
/* exact_only */ true, SC_PTR_BITS, /* prefer_small */ false);
expect_ptr_eq(&exact, fit,
"Exact search should enumerate the floor bin when large size "
"classes are disabled");
fit = eset_fit(&exact_eset, request - PAGE, PAGE,
/* exact_only */ true, SC_PTR_BITS, /* prefer_small */ false);
expect_ptr_null(fit,
"Exact search should not return merely larger extents");
}
TEST_END
int
main(void) {
return test_no_reentrancy(test_eset_insert_remove_fit,
test_eset_alignment_and_large_class_fallback,
test_eset_exact_fit_large_class_disabled);
}

278
test/unit/hpa_central.c Normal file
View file

@ -0,0 +1,278 @@
#include "test/jemalloc_test.h"
#include "jemalloc/internal/hpa.h"
#include "jemalloc/internal/hpa_central.h"
#define HPA_TEST_ARENA_IND 125
#define HPA_TEST_EDEN_SIZE (128 * HUGEPAGE)
static bool hpa_test_map_fail;
static unsigned hpa_test_map_calls;
static unsigned hpa_test_unmap_calls;
static unsigned hpa_test_hugify_calls;
static void *hpa_test_last_map;
static size_t hpa_test_last_map_size;
static void *hpa_test_last_unmap;
static size_t hpa_test_last_unmap_size;
static void *hpa_test_last_hugify;
static size_t hpa_test_last_hugify_size;
static void
hpa_test_hooks_reset(void) {
hpa_test_map_fail = false;
hpa_test_map_calls = 0;
hpa_test_unmap_calls = 0;
hpa_test_hugify_calls = 0;
hpa_test_last_map = NULL;
hpa_test_last_map_size = 0;
hpa_test_last_unmap = NULL;
hpa_test_last_unmap_size = 0;
hpa_test_last_hugify = NULL;
hpa_test_last_hugify_size = 0;
}
static void *
hpa_test_map(size_t size) {
hpa_test_map_calls++;
hpa_test_last_map_size = size;
if (hpa_test_map_fail) {
hpa_test_last_map = NULL;
return NULL;
}
bool commit = true;
void *ret = pages_map(NULL, size, HUGEPAGE, &commit);
assert_true(commit, "HPA test mappings should be committed");
hpa_test_last_map = ret;
return ret;
}
static void
hpa_test_unmap(void *ptr, size_t size) {
hpa_test_unmap_calls++;
hpa_test_last_unmap = ptr;
hpa_test_last_unmap_size = size;
pages_unmap(ptr, size);
}
static void
hpa_test_purge(void *ptr, size_t size) {
}
static bool
hpa_test_hugify(void *ptr, size_t size, bool sync) {
hpa_test_hugify_calls++;
hpa_test_last_hugify = ptr;
hpa_test_last_hugify_size = size;
return false;
}
static void
hpa_test_dehugify(void *ptr, size_t size) {
}
static void
hpa_test_curtime(nstime_t *r_time, bool first_reading) {
nstime_init(r_time, 0);
}
static uint64_t
hpa_test_ms_since(nstime_t *r_time) {
return 0;
}
static bool
hpa_test_vectorized_purge(void *vec, size_t vlen, size_t nbytes) {
return true;
}
static hpa_hooks_t hpa_test_hooks = {
hpa_test_map,
hpa_test_unmap,
hpa_test_purge,
hpa_test_hugify,
hpa_test_dehugify,
hpa_test_curtime,
hpa_test_ms_since,
hpa_test_vectorized_purge
};
static bool hpa_base_fail_after_new;
static unsigned hpa_base_alloc_calls;
static void
hpa_base_hooks_reset(void) {
hpa_base_fail_after_new = false;
hpa_base_alloc_calls = 0;
}
static void *
hpa_base_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
if (hpa_base_fail_after_new && hpa_base_alloc_calls > 0) {
hpa_base_alloc_calls++;
return NULL;
}
hpa_base_alloc_calls++;
return pages_map(new_addr, size, alignment, commit);
}
static bool
hpa_base_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind) {
pages_unmap(addr, size);
return false;
}
static void
hpa_base_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind) {
pages_unmap(addr, size);
}
static extent_hooks_t hpa_base_hooks = {
hpa_base_alloc,
hpa_base_dalloc,
hpa_base_destroy,
NULL, /* commit */
NULL, /* decommit */
NULL, /* purge_lazy */
NULL, /* purge_forced */
NULL, /* split */
NULL /* merge */
};
static hpdata_t *
hpa_central_extract_with_lock(hpa_central_t *central, malloc_mutex_t *mtx,
uint64_t age, bool hugify_eager, bool *oom) {
tsdn_t *tsdn = tsdn_fetch();
malloc_mutex_lock(tsdn, mtx);
hpdata_t *ret = hpa_central_extract(tsdn, central, PAGE, age,
hugify_eager, oom);
malloc_mutex_unlock(tsdn, mtx);
return ret;
}
static void
hpa_test_shard_grow_mtx_init(malloc_mutex_t *mtx) {
assert_false(malloc_mutex_init(mtx, "hpa_test_shard_grow",
WITNESS_RANK_HPA_SHARD_GROW, malloc_mutex_rank_exclusive),
"Unexpected mutex initialization failure");
}
TEST_BEGIN(test_hpa_central_extract_eden) {
test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit());
hpa_test_hooks_reset();
tsdn_t *tsdn = tsdn_fetch();
base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(base, "Unexpected base_new failure");
hpa_central_t central;
assert_false(hpa_central_init(&central, base, &hpa_test_hooks),
"Unexpected hpa_central_init failure");
malloc_mutex_t shard_grow_mtx;
hpa_test_shard_grow_mtx_init(&shard_grow_mtx);
void *eden = NULL;
for (unsigned i = 0; i < 128; i++) {
bool oom = true;
hpdata_t *ps = hpa_central_extract_with_lock(&central,
&shard_grow_mtx, 1000 + i, /* hugify_eager */ true, &oom);
expect_false(oom, "Unexpected HPA central OOM");
expect_ptr_not_null(ps, "Unexpected HPA central extraction failure");
if (i == 0) {
eden = hpa_test_last_map;
expect_u_eq(1, hpa_test_map_calls,
"First extraction should map eden");
expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_map_size,
"Unexpected eden mapping size");
expect_u_eq(1, hpa_test_hugify_calls,
"Eager extraction should hugify the whole eden");
expect_ptr_eq(eden, hpa_test_last_hugify,
"Hugify should apply to eden");
expect_zu_eq(HPA_TEST_EDEN_SIZE,
hpa_test_last_hugify_size,
"Hugify should cover the whole eden");
}
expect_ptr_eq((void *)((byte_t *)eden + i * HUGEPAGE),
hpdata_addr_get(ps), "Unexpected extracted pageslab addr");
expect_u64_eq(1000 + i, hpdata_age_get(ps),
"Unexpected hpdata age");
expect_true(hpdata_huge_get(ps),
"Eager extraction should mark hpdata huge");
}
expect_ptr_null(central.eden, "Exact final extraction should empty eden");
expect_zu_eq(0, central.eden_len,
"Exact final extraction should clear eden length");
expect_u_eq(1, hpa_test_map_calls,
"All pageslabs should come from one eden mapping");
expect_u_eq(0, hpa_test_unmap_calls,
"Successful extraction should not unmap eden");
pages_unmap(eden, HPA_TEST_EDEN_SIZE);
base_delete(tsdn, base);
}
TEST_END
TEST_BEGIN(test_hpa_central_failure_paths) {
test_skip_if(!hpa_supported() || hpa_hugepage_size_exceeds_limit());
hpa_test_hooks_reset();
tsdn_t *tsdn = tsdn_fetch();
base_t *base = base_new(tsdn, HPA_TEST_ARENA_IND + 1,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(base, "Unexpected base_new failure");
hpa_central_t central;
assert_false(hpa_central_init(&central, base, &hpa_test_hooks),
"Unexpected hpa_central_init failure");
malloc_mutex_t shard_grow_mtx;
hpa_test_shard_grow_mtx_init(&shard_grow_mtx);
hpa_test_map_fail = true;
bool oom = false;
hpdata_t *ps = hpa_central_extract_with_lock(&central,
&shard_grow_mtx, 1, /* hugify_eager */ false, &oom);
expect_ptr_null(ps, "Map failure should not return hpdata");
expect_true(oom, "Map failure should report OOM");
expect_u_eq(1, hpa_test_map_calls, "Expected one map attempt");
expect_u_eq(0, hpa_test_unmap_calls,
"Map failure should not call unmap");
base_delete(tsdn, base);
hpa_base_hooks_reset();
hpa_test_hooks_reset();
base = base_new(tsdn, HPA_TEST_ARENA_IND + 2, &hpa_base_hooks,
/* metadata_use_hooks */ true);
assert_ptr_not_null(base, "Unexpected base_new failure");
hpa_base_fail_after_new = true;
while (base_alloc(tsdn, base, sizeof(hpdata_t), CACHELINE) != NULL) {
}
assert_false(hpa_central_init(&central, base, &hpa_test_hooks),
"Unexpected hpa_central_init failure");
malloc_mutex_t shard_grow_mtx2;
hpa_test_shard_grow_mtx_init(&shard_grow_mtx2);
oom = false;
ps = hpa_central_extract_with_lock(&central, &shard_grow_mtx2, 2,
/* hugify_eager */ false, &oom);
expect_ptr_null(ps, "Metadata OOM should not return hpdata");
expect_true(oom, "Metadata allocation failure should report OOM");
expect_u_eq(1, hpa_test_map_calls,
"Metadata OOM should happen after mapping eden once");
expect_u_eq(1, hpa_test_unmap_calls,
"Metadata OOM should unmap the freshly mapped eden");
expect_ptr_eq(hpa_test_last_map, hpa_test_last_unmap,
"Metadata OOM should unmap the eden it just mapped");
expect_zu_eq(HPA_TEST_EDEN_SIZE, hpa_test_last_unmap_size,
"Metadata OOM should unmap the full eden");
base_delete(tsdn, base);
}
TEST_END
int
main(void) {
return test_no_reentrancy(test_hpa_central_extract_eden,
test_hpa_central_failure_paths);
}

395
test/unit/pac.c Normal file
View file

@ -0,0 +1,395 @@
#include "test/jemalloc_test.h"
#include "jemalloc/internal/pa.h"
#define PAC_TEST_ARENA_IND 124
static bool pac_test_dalloc_fail;
static bool pac_test_purge_lazy_fail;
static unsigned pac_test_alloc_calls;
static unsigned pac_test_dalloc_calls;
static unsigned pac_test_destroy_calls;
static unsigned pac_test_purge_lazy_calls;
static void
pac_test_hooks_reset(void) {
pac_test_dalloc_fail = false;
pac_test_purge_lazy_fail = false;
pac_test_alloc_calls = 0;
pac_test_dalloc_calls = 0;
pac_test_destroy_calls = 0;
pac_test_purge_lazy_calls = 0;
}
static void *
pac_test_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size,
size_t alignment, bool *zero, bool *commit, unsigned arena_ind) {
pac_test_alloc_calls++;
void *ret = pages_map(new_addr, size, alignment, commit);
return ret;
}
static bool
pac_test_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind) {
pac_test_dalloc_calls++;
if (pac_test_dalloc_fail) {
return true;
}
pages_unmap(addr, size);
return false;
}
static void
pac_test_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size,
bool committed, unsigned arena_ind) {
pac_test_destroy_calls++;
pages_unmap(addr, size);
}
static bool
pac_test_purge_lazy(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t offset, size_t length, unsigned arena_ind) {
pac_test_purge_lazy_calls++;
return pac_test_purge_lazy_fail;
}
static bool
pac_test_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {
return !maps_coalesce && !opt_retain;
}
static bool
pac_test_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
return !maps_coalesce && !opt_retain;
}
static extent_hooks_t pac_test_hooks = {
pac_test_alloc,
pac_test_dalloc,
pac_test_destroy,
NULL, /* commit */
NULL, /* decommit */
pac_test_purge_lazy,
NULL, /* purge_forced */
pac_test_split,
pac_test_merge
};
typedef struct pac_test_data_s pac_test_data_t;
struct pac_test_data_s {
pa_shard_t shard;
pa_central_t central;
base_t *base;
emap_t emap;
pa_shard_stats_t stats;
malloc_mutex_t stats_mtx;
extent_hooks_t hooks;
};
static pac_test_data_t *
pac_test_data_init(bool custom_hooks, ssize_t dirty_decay_ms,
ssize_t muzzy_decay_ms) {
tsdn_t *tsdn = tsdn_fetch();
pac_test_data_t *data = calloc(1, sizeof(*data));
assert_ptr_not_null(data, "Unexpected calloc failure");
if (custom_hooks) {
memcpy(&data->hooks, &pac_test_hooks, sizeof(extent_hooks_t));
} else {
memcpy(&data->hooks, &ehooks_default_extent_hooks,
sizeof(extent_hooks_t));
}
data->base = base_new(tsdn, PAC_TEST_ARENA_IND, &data->hooks,
/* metadata_use_hooks */ true);
assert_ptr_not_null(data->base, "Unexpected base_new failure");
assert_false(emap_init(&data->emap, data->base, /* zeroed */ true),
"Unexpected emap_init failure");
assert_false(malloc_mutex_init(&data->stats_mtx, "pac_test_stats",
WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive),
"Unexpected stats mutex initialization failure");
nstime_t time;
nstime_init(&time, 0);
assert_false(pa_central_init(&data->central, data->base, /* hpa */ false,
&hpa_hooks_default), "Unexpected pa_central_init failure");
assert_false(pa_shard_init(tsdn, &data->shard, &data->central,
&data->emap, data->base, PAC_TEST_ARENA_IND, &data->stats,
&data->stats_mtx, &time, SC_LARGE_MAXCLASS + PAGE,
dirty_decay_ms, muzzy_decay_ms), "Unexpected pa_shard_init failure");
return data;
}
static void
pac_decay_all_locked(pac_t *pac, extent_state_t state, bool fully_decay) {
decay_t *decay;
pac_decay_stats_t *decay_stats;
ecache_t *ecache;
if (state == extent_state_dirty) {
decay = &pac->decay_dirty;
decay_stats = &pac->stats->decay_dirty;
ecache = &pac->ecache_dirty;
} else {
assert_d_eq(extent_state_muzzy, state,
"Only dirty and muzzy decay are supported");
decay = &pac->decay_muzzy;
decay_stats = &pac->stats->decay_muzzy;
ecache = &pac->ecache_muzzy;
}
tsdn_t *tsdn = tsdn_fetch();
malloc_mutex_lock(tsdn, &decay->mtx);
pac_decay_all(tsdn, pac, decay, decay_stats, ecache, fully_decay);
malloc_mutex_unlock(tsdn, &decay->mtx);
}
static void
pac_test_data_destroy(pac_test_data_t *data) {
/*
* Decay below calls back into the test hooks; reset all hook state
* (including the fail flags) so teardown is unaffected by anything the
* preceding test toggled.
*/
pac_test_hooks_reset();
pac_decay_all_locked(&data->shard.pac, extent_state_dirty,
/* fully_decay */ true);
pac_decay_all_locked(&data->shard.pac, extent_state_muzzy,
/* fully_decay */ true);
pa_shard_destroy(tsdn_fetch(), &data->shard);
base_delete(tsdn_fetch(), data->base);
/*
* pac operations populated the tsd rtree_ctx with leaf-node pointers
* from the private emap we just destroyed. Invalidate the cache so
* the next test's fresh emap doesn't follow stale entries.
*/
rtree_ctx_data_init(tsd_rtree_ctx(tsd_fetch()));
free(data);
}
static edata_t *
pac_alloc_expect(pac_test_data_t *data, size_t size, bool guarded) {
bool deferred_work_generated = false;
edata_t *edata = pac_alloc(tsdn_fetch(), &data->shard.pac, size, PAGE,
/* zero */ false, guarded, /* frequent_reuse */ false,
&deferred_work_generated);
expect_ptr_not_null(edata, "Unexpected pac_alloc failure");
expect_zu_eq(size, edata_size_get(edata), "Unexpected allocation size");
return edata;
}
TEST_BEGIN(test_pac_dirty_muzzy_alloc_priority) {
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
pac_t *pac = &data->shard.pac;
size_t alloc_size = HUGEPAGE;
size_t alloc_npages = alloc_size >> LG_PAGE;
edata_t *muzzy = pac_alloc_expect(
data, alloc_size, /* guarded */ false);
void *muzzy_addr = edata_base_get(muzzy);
edata_t *dirty = pac_alloc_expect(
data, alloc_size, /* guarded */ false);
void *dirty_addr = edata_base_get(dirty);
bool deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, muzzy, &deferred_work_generated);
pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false);
expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_muzzy),
"Expected one muzzy page after dirty decay");
deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, dirty, &deferred_work_generated);
expect_zu_eq(alloc_npages, ecache_npages_get(&pac->ecache_dirty),
"Expected one dirty page");
edata_t *from_dirty = pac_alloc_expect(
data, alloc_size, /* guarded */ false);
expect_ptr_eq(dirty_addr, edata_base_get(from_dirty),
"Dirty cache should be preferred over muzzy");
edata_t *from_muzzy = pac_alloc_expect(
data, alloc_size, /* guarded */ false);
expect_ptr_eq(muzzy_addr, edata_base_get(from_muzzy),
"Muzzy cache should be used after dirty cache");
deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, from_dirty, &deferred_work_generated);
deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, from_muzzy, &deferred_work_generated);
pac_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_pac_batched_grow_caches_trailing_dirty) {
test_skip_if(!sz_large_size_classes_disabled()
|| !(maps_coalesce || opt_retain));
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
pac_t *pac = &data->shard.pac;
size_t size = HUGEPAGE + PAGE;
size_t batched_size = sz_s2u_compute_using_delta(size);
size_t next_hugepage_size = HUGEPAGE_CEILING(size);
if (batched_size > next_hugepage_size) {
batched_size = next_hugepage_size;
}
assert_zu_gt(batched_size, size,
"Test size should exercise batched retained growth");
size_t dirty_before = ecache_npages_get(&pac->ecache_dirty);
edata_t *large = pac_alloc_expect(data, size, /* guarded */ false);
expect_zu_eq(dirty_before + ((batched_size - size) >> LG_PAGE),
ecache_npages_get(&pac->ecache_dirty),
"Batched grow should cache the trailing dirty extent");
if (config_stats) {
expect_zu_ge(pac_mapped(pac), batched_size,
"Mapped stats should include the full batched grow");
}
bool deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, large, &deferred_work_generated);
pac_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_pac_deferred_work_signals) {
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
pac_t *pac = &data->shard.pac;
edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false);
bool deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated);
expect_true(deferred_work_generated,
"Non-pinned dalloc should request deferred work");
pac_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_pac_decay_dirty_to_muzzy_via_purge_lazy) {
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
pac_t *pac = &data->shard.pac;
edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false);
bool deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated);
unsigned purge_lazy_before = pac_test_purge_lazy_calls;
pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ false);
expect_zu_eq(0, ecache_npages_get(&pac->ecache_dirty),
"Dirty decay should remove dirty pages");
expect_zu_eq(1, ecache_npages_get(&pac->ecache_muzzy),
"Successful lazy purge should move dirty pages to muzzy");
expect_u_gt(pac_test_purge_lazy_calls, purge_lazy_before,
"Dirty-to-muzzy decay should call purge_lazy");
pac_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_pac_decay_retains_when_dalloc_fails) {
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
pac_t *pac = &data->shard.pac;
pac_test_dalloc_fail = true;
edata_t *edata = pac_alloc_expect(data, PAGE, /* guarded */ false);
bool deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, edata, &deferred_work_generated);
size_t retained_before = ecache_npages_get(&pac->ecache_retained);
pac_decay_all_locked(pac, extent_state_dirty, /* fully_decay */ true);
expect_zu_gt(ecache_npages_get(&pac->ecache_retained), retained_before,
"Fully decayed dirty pages should be retained when dalloc fails");
expect_u_gt(pac_test_dalloc_calls, 0,
"Fully decayed dirty pages should attempt dalloc first");
pac_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_pac_non_pinned_dalloc_signals_deferred_work) {
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
pac_t *pac = &data->shard.pac;
size_t normal_size = 3 * HUGEPAGE + PAGE;
edata_t *normal = pac_alloc_expect(data, normal_size,
/* guarded */ false);
bool deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated);
expect_true(deferred_work_generated,
"Non-pinned dalloc should request deferred work");
pac_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_pac_non_pinned_shrink_signals_deferred_work) {
test_skip_if(!maps_coalesce);
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
pac_t *pac = &data->shard.pac;
size_t normal_size = 3 * HUGEPAGE + PAGE;
size_t normal_shrink_size = 3 * HUGEPAGE;
edata_t *normal = pac_alloc_expect(data, normal_size,
/* guarded */ false);
bool deferred_work_generated = false;
expect_false(pac_shrink(tsdn_fetch(), pac, normal, normal_size,
normal_shrink_size, &deferred_work_generated),
"Unexpected non-pinned shrink failure");
expect_true(deferred_work_generated,
"Non-pinned shrink should request deferred work");
pac_dalloc(tsdn_fetch(), pac, normal, &deferred_work_generated);
pac_test_data_destroy(data);
}
TEST_END
TEST_BEGIN(test_pac_large_guarded_dalloc_unguards_before_caching) {
pac_test_hooks_reset();
pac_test_data_t *data = pac_test_data_init(
/* custom_hooks */ true, -1, -1);
edata_t *guarded = pac_alloc_expect(data, SC_LARGE_MINCLASS,
/* guarded */ true);
expect_true(edata_guarded_get(guarded),
"Guarded allocation should set the guarded bit");
bool deferred_work_generated = false;
pac_dalloc(tsdn_fetch(), &data->shard.pac, guarded,
&deferred_work_generated);
expect_true(deferred_work_generated,
"Guarded dalloc should still request deferred work");
expect_false(edata_guarded_get(guarded),
"Large guarded dalloc should unguard before caching");
pac_test_data_destroy(data);
}
TEST_END
int
main(void) {
return test_no_reentrancy(test_pac_dirty_muzzy_alloc_priority,
test_pac_batched_grow_caches_trailing_dirty,
test_pac_deferred_work_signals,
test_pac_decay_dirty_to_muzzy_via_purge_lazy,
test_pac_decay_retains_when_dalloc_fails,
test_pac_non_pinned_dalloc_signals_deferred_work,
test_pac_non_pinned_shrink_signals_deferred_work,
test_pac_large_guarded_dalloc_unguards_before_caching);
}

166
test/unit/tcache_gc.c Normal file
View file

@ -0,0 +1,166 @@
#include "test/jemalloc_test.h"
extern cache_bin_sz_t tcache_gc_small_nremote_get_test(
cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min,
uintptr_t *addr_max, szind_t szind, size_t nflush);
extern void tcache_gc_small_bin_shuffle_test(cache_bin_t *cache_bin,
cache_bin_sz_t nremote, uintptr_t addr_min, uintptr_t addr_max);
extern uint8_t tcache_nfill_small_lg_div_get_test(
tcache_slow_t *tcache_slow, szind_t szind);
extern void tcache_nfill_small_burst_prepare_test(
tcache_slow_t *tcache_slow, szind_t szind);
extern void tcache_nfill_small_burst_reset_test(
tcache_slow_t *tcache_slow, szind_t szind);
extern void tcache_nfill_small_gc_update_test(
tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit);
extern uint8_t tcache_gc_item_delay_compute_test(szind_t szind);
static void *
test_cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info,
cache_bin_sz_t ncached_max) {
cache_bin_info_init(info, ncached_max);
size_t size;
size_t alignment;
cache_bin_info_compute_alloc(info, 1, &size, &alignment);
void *mem = mallocx(size, MALLOCX_ALIGN(alignment));
assert_ptr_not_null(mem, "Unexpected mallocx failure");
size_t cur_offset = 0;
cache_bin_preincrement(info, 1, mem, &cur_offset);
cache_bin_init(bin, info, mem, &cur_offset);
cache_bin_postincrement(mem, &cur_offset);
assert_zu_eq(cur_offset, size, "Should use all requested memory");
return mem;
}
static void
cache_bin_fill_ptrs(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill) {
CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill);
cache_bin_init_ptr_array_for_fill(bin, &arr, nfill);
for (cache_bin_sz_t i = 0; i < nfill; i++) {
arr.ptr[i] = ptrs[i];
}
cache_bin_finish_fill(bin, &arr, nfill);
expect_zu_eq(nfill, cache_bin_ncached_get_local(bin),
"Unexpected fill count");
}
TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) {
cache_bin_t bin;
cache_bin_info_t info;
void *mem = test_cache_bin_init(&bin, &info, 16);
szind_t szind = 0;
uintptr_t anchor = ZU(0x40000000);
size_t slab_size = bin_infos[szind].slab_size;
void *ptrs[] = {
(void *)(anchor + 16),
(void *)(anchor + slab_size + 16),
(void *)(anchor + 64),
(void *)(anchor + TCACHE_GC_NEIGHBOR_LIMIT + PAGE),
};
cache_bin_fill_ptrs(&bin, ptrs, 4);
uintptr_t addr_min;
uintptr_t addr_max;
cache_bin_sz_t nremote = tcache_gc_small_nremote_get_test(&bin,
(void *)anchor, &addr_min, &addr_max, szind, 2);
expect_zu_eq(2, nremote,
"Should count pointers outside the local slab");
expect_zu_eq(anchor, addr_min, "Expected slab-local lower bound");
expect_zu_eq(anchor + slab_size, addr_max,
"Expected slab-local upper bound");
tcache_gc_small_bin_shuffle_test(&bin, nremote, addr_min, addr_max);
expect_ptr_eq(ptrs[0], bin.stack_head[0],
"Local pointer order should be preserved");
expect_ptr_eq(ptrs[2], bin.stack_head[1],
"Local pointer order should be preserved");
for (unsigned i = 2; i < 4; i++) {
expect_true((uintptr_t)bin.stack_head[i] < addr_min
|| (uintptr_t)bin.stack_head[i] >= addr_max,
"Remote pointers should be moved to the flush side");
}
while (cache_bin_ncached_get_local(&bin) > 0) {
bool success;
cache_bin_alloc(&bin, &success);
}
cache_bin_fill_ptrs(&bin, ptrs, 4);
nremote = tcache_gc_small_nremote_get_test(&bin, (void *)anchor,
&addr_min, &addr_max, szind, 1);
expect_zu_eq(1, nremote,
"Neighbor filtering should be used when it satisfies nflush");
expect_zu_eq(anchor - TCACHE_GC_NEIGHBOR_LIMIT, addr_min,
"Expected neighbor lower bound");
expect_zu_eq(anchor + TCACHE_GC_NEIGHBOR_LIMIT, addr_max,
"Expected neighbor upper bound");
free(mem);
}
TEST_END
TEST_BEGIN(test_tcache_gc_fill_control_and_delay) {
tcache_slow_t tcache_slow;
memset(&tcache_slow, 0, sizeof(tcache_slow));
szind_t szind = 0;
cache_bin_fill_ctl_t *ctl =
&tcache_slow.bin_fill_ctl_do_not_access_directly[szind];
ctl->base = 3;
ctl->offset = 0;
bool old_experimental_tcache_gc = opt_experimental_tcache_gc;
size_t old_tcache_gc_delay_bytes = opt_tcache_gc_delay_bytes;
opt_experimental_tcache_gc = true;
expect_u_eq(3, tcache_nfill_small_lg_div_get_test(
&tcache_slow, szind), "Unexpected initial fill divisor");
tcache_nfill_small_burst_prepare_test(&tcache_slow, szind);
expect_u_eq(2, tcache_nfill_small_lg_div_get_test(
&tcache_slow, szind), "Burst load should increase fill count");
tcache_nfill_small_burst_prepare_test(&tcache_slow, szind);
expect_u_eq(1, tcache_nfill_small_lg_div_get_test(
&tcache_slow, szind), "Burst load should cap at divisor 1");
tcache_nfill_small_burst_prepare_test(&tcache_slow, szind);
expect_u_eq(1, tcache_nfill_small_lg_div_get_test(
&tcache_slow, szind), "Burst offset should not reach base");
tcache_nfill_small_burst_reset_test(&tcache_slow, szind);
expect_u_eq(3, tcache_nfill_small_lg_div_get_test(
&tcache_slow, szind), "Burst reset should clear offset");
tcache_nfill_small_gc_update_test(&tcache_slow, szind, 0);
expect_u_eq(2, ctl->base,
"Refill during a GC period should increase future fill count");
expect_u_eq(0, ctl->offset, "GC update should reset burst offset");
tcache_nfill_small_gc_update_test(&tcache_slow, szind, 64);
expect_u_eq(3, ctl->base,
"Low-water pressure should reduce future fill count");
ctl->offset = 2;
opt_experimental_tcache_gc = false;
expect_u_eq(3, tcache_nfill_small_lg_div_get_test(
&tcache_slow, szind), "Legacy GC should ignore burst offset");
size_t sz = sz_index2size(szind);
opt_tcache_gc_delay_bytes = 3 * sz;
expect_u_eq(3, tcache_gc_item_delay_compute_test(szind),
"Delay should convert bytes to items");
opt_tcache_gc_delay_bytes = SIZE_T_MAX;
expect_u_eq(UINT8_MAX, tcache_gc_item_delay_compute_test(szind),
"Delay should saturate at uint8 max");
opt_experimental_tcache_gc = old_experimental_tcache_gc;
opt_tcache_gc_delay_bytes = old_tcache_gc_delay_bytes;
}
TEST_END
int
main(void) {
return test_no_reentrancy(test_tcache_gc_small_remote_count_and_shuffle,
test_tcache_gc_fill_control_and_delay);
}