romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-07 22:37:16 +03:00
Code Issues Projects Releases Packages Wiki Activity

Move malloc routing into new malloc_dispatch module

Browse source 
Pull the tcache-aware routing helpers out of arena into a layer that
sits directly below the public malloc interface:

  arena_malloc           -> malloc_dispatch_malloc
  arena_palloc           -> malloc_dispatch_palloc
  arena_ralloc           -> malloc_dispatch_ralloc
  arena_dalloc*          -> malloc_dispatch_dalloc*
  arena_sdalloc*         -> malloc_dispatch_sdalloc*
  arena_dalloc_promoted  -> malloc_dispatch_dalloc_promoted

The new module (malloc_dispatch.h, malloc_dispatch_inlines.h,
src/malloc_dispatch.c) owns the tcache-vs-fall-through decision; the
only consumer is jemalloc_internal_inlines_c.h.  arena keeps a narrower
arena_prof_demote() for the sampled-allocation demotion path.
This commit is contained in:
Slobodan Predolac 2026-06-02 09:42:36 -07:00
parent 5543c262c7
commit 9d75722344
18 changed files with 649 additions and 385 deletions
Download patch file Download diff file Expand all files Collapse all files

208
test/unit/malloc_dispatch.c Normal file
View file

@ -0,0 +1,208 @@
#include "test/jemalloc_test.h"
/*
* Targeted coverage for the malloc_dispatch routing layer. Integration
* tests exercise the public API end-to-end but don't assert which branch
* was taken for a given (size, tcache) input. These tests make the
* routing observable through per-arena stats counters:
*
* 1. small + tcache != NULL -> tcache_alloc_small
* 2. small + tcache == NULL -> arena_malloc_hard + arena_dalloc_small
* 3. large <= tcache_max + tcache -> tcache_alloc_large
* 4. large > tcache_max + tcache -> arena_malloc_hard + large_dalloc
*/
static unsigned
create_fresh_arena(void) {
unsigned arena_ind;
size_t sz = sizeof(arena_ind);
expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
0, "arenas.create failed");
return arena_ind;
}
static void
refresh_stats(void) {
uint64_t epoch = 1;
expect_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
"epoch refresh failed");
}
static uint64_t
read_u64_mallctl(const char *cmd) {
uint64_t v;
size_t sz = sizeof(v);
expect_d_eq(mallctl(cmd, (void *)&v, &sz, NULL, 0), 0,
"mallctl read failed");
return v;
}
static uint64_t
arena_bin0_nmalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.bins.0.nmalloc", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_bin0_ndalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.bins.0.ndalloc", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_bin0_nfills(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.bins.0.nfills", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_large_nmalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.large.nmalloc", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_large_ndalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.large.ndalloc", arena_ind);
return read_u64_mallctl(cmd);
}
/*
* Branch 2: small alloc/dalloc with MALLOCX_TCACHE_NONE must increment the
* arena bin counters immediately (no tcache to absorb the call).
*/
TEST_BEGIN(test_dispatch_small_no_tcache) {
test_skip_if(!config_stats);
test_skip_if(opt_prof);
unsigned arena_ind = create_fresh_arena();
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
size_t sz = bin_infos[0].reg_size;
refresh_stats();
uint64_t nmalloc_before = arena_bin0_nmalloc(arena_ind);
uint64_t ndalloc_before = arena_bin0_ndalloc(arena_ind);
uint64_t nfills_before = arena_bin0_nfills(arena_ind);
void *p = mallocx(sz, flags);
expect_ptr_not_null(p, "mallocx failed");
dallocx(p, flags);
refresh_stats();
expect_u64_eq(arena_bin0_nmalloc(arena_ind), nmalloc_before + 1,
"small no-tcache alloc must increment arena bin nmalloc");
expect_u64_eq(arena_bin0_ndalloc(arena_ind), ndalloc_before + 1,
"small no-tcache dalloc must increment arena bin ndalloc");
expect_u64_eq(arena_bin0_nfills(arena_ind), nfills_before,
"no-tcache path must not trigger any tcache fill");
}
TEST_END
/*
* Branch 1: small alloc with tcache must NOT increment the arena bin nmalloc
* counter directly (the tcache absorbs the alloc); a refill (nfills > 0) is
* the routing observable instead. Bound the test to a single tcache slot to
* make the refill predictable.
*/
TEST_BEGIN(test_dispatch_small_with_tcache) {
test_skip_if(!config_stats);
test_skip_if(!opt_tcache);
test_skip_if(opt_prof);
unsigned arena_ind = create_fresh_arena();
int flags = MALLOCX_ARENA(arena_ind);
size_t sz = bin_infos[0].reg_size;
/* Flush any per-thread tcache so this arena's tcache slot starts cold. */
expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0,
"thread.tcache.flush failed");
refresh_stats();
uint64_t nfills_before = arena_bin0_nfills(arena_ind);
void *p = mallocx(sz, flags);
expect_ptr_not_null(p, "mallocx failed");
refresh_stats();
expect_u64_gt(arena_bin0_nfills(arena_ind), nfills_before,
"small alloc with tcache must trigger a tcache fill");
dallocx(p, flags);
}
TEST_END
/*
* Branches 3 & 4: large alloc routing pivots on tcache_max. Use a fresh
* thread.tcache.max so we can compute a size that is just above it; that size
* must be routed through arena_malloc_hard (and not the tcache). Allocations
* below tcache_max routed through the same call site must not touch the arena
* large counters on the malloc/free pair (they ride the tcache).
*/
TEST_BEGIN(test_dispatch_large_routes_on_tcache_max) {
test_skip_if(!config_stats);
test_skip_if(!opt_tcache);
test_skip_if(opt_prof);
test_skip_if(san_uaf_detection_enabled());
unsigned arena_ind = create_fresh_arena();
expect_d_eq(
mallctl("thread.arena", NULL, NULL, &arena_ind, sizeof(arena_ind)),
0, "thread.arena bind failed");
/* Pin tcache_max to a known small-large boundary. */
size_t small_large;
size_t sz = sizeof(small_large);
expect_d_eq(mallctl("arenas.lextent.0.size", (void *)&small_large, &sz,
NULL, 0),
0, "arenas.lextent.0.size lookup failed");
expect_d_eq(mallctl("thread.tcache.max", NULL, NULL, &small_large,
sizeof(small_large)),
0, "thread.tcache.max set failed");
expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0,
"thread.tcache.flush failed");
int flags = MALLOCX_ARENA(arena_ind);
/* Above tcache_max: must hit arena_malloc_hard + large_dalloc. */
size_t too_big;
expect_d_eq(mallctl("arenas.lextent.1.size", (void *)&too_big, &sz, NULL,
0),
0, "arenas.lextent.1.size lookup failed");
expect_zu_gt(too_big, small_large,
"lextent.1 must exceed tcache_max boundary");
refresh_stats();
uint64_t nmalloc_before = arena_large_nmalloc(arena_ind);
uint64_t ndalloc_before = arena_large_ndalloc(arena_ind);
void *big = mallocx(too_big, flags);
expect_ptr_not_null(big, "mallocx failed");
refresh_stats();
expect_u64_eq(arena_large_nmalloc(arena_ind), nmalloc_before + 1,
"large alloc above tcache_max must bypass tcache "
"and increment arena large nmalloc");
dallocx(big, flags);
refresh_stats();
expect_u64_eq(arena_large_ndalloc(arena_ind), ndalloc_before + 1,
"large dalloc above tcache_max must bypass tcache "
"and increment arena large ndalloc");
}
TEST_END
int
main(void) {
return test(test_dispatch_small_no_tcache,
test_dispatch_small_with_tcache,
test_dispatch_large_routes_on_tcache_max);
}