romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-06-13 23:45:46 +03:00
Code Issues Projects Releases Packages Wiki Activity

Move malloc routing into new malloc_dispatch module

Browse source 
Pull the tcache-aware routing helpers out of arena into a layer that
sits directly below the public malloc interface:

  arena_malloc           -> malloc_dispatch_malloc
  arena_palloc           -> malloc_dispatch_palloc
  arena_ralloc           -> malloc_dispatch_ralloc
  arena_dalloc*          -> malloc_dispatch_dalloc*
  arena_sdalloc*         -> malloc_dispatch_sdalloc*
  arena_dalloc_promoted  -> malloc_dispatch_dalloc_promoted

The new module (malloc_dispatch.h, malloc_dispatch_inlines.h,
src/malloc_dispatch.c) owns the tcache-vs-fall-through decision; the
only consumer is jemalloc_internal_inlines_c.h.  arena keeps a narrower
arena_prof_demote() for the sampled-allocation demotion path.
This commit is contained in:
Slobodan Predolac 2026-06-02 09:42:36 -07:00
parent 5543c262c7
commit 9d75722344
18 changed files with 649 additions and 385 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Makefile.in
View file

@ -132,6 +132,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/inspect.c \
$(srcroot)src/large.c \
$(srcroot)src/log.c \
$(srcroot)src/malloc_dispatch.c \
$(srcroot)src/malloc_io.c \
$(srcroot)src/conf.c \
$(srcroot)src/mutex.c \
@ -260,6 +261,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/log.c \
$(srcroot)test/unit/mallctl.c \
$(srcroot)test/unit/malloc_conf_2.c \
$(srcroot)test/unit/malloc_dispatch.c \
$(srcroot)test/unit/malloc_io.c \
$(srcroot)test/unit/math.c \
$(srcroot)test/unit/mpsc_queue.c \

7
include/jemalloc/internal/arena_externs.h
View file

@ -70,12 +70,9 @@ cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, bool slab);
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero, bool slab, tcache_t *tcache);
void arena_prof_promote(
tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
void arena_dalloc_promoted(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
@ -84,8 +81,6 @@ void arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
arena_t *stats_arena, cache_bin_stats_t merge_stats);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero, size_t *newsize);
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache);
dss_prec_t arena_dss_prec_get(const arena_t *arena);
ehooks_t *arena_get_ehooks(const arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(

259
include/jemalloc/internal/arena_inlines_b.h
View file

@ -17,7 +17,6 @@
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache_inlines.h"
#include "jemalloc/internal/ticker.h"
static inline arena_t *
@ -193,26 +192,6 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
arena_decay_ticks(tsdn, arena, 1);
}
JEMALLOC_ALWAYS_INLINE void *
arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
bool slab, tcache_t *tcache, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
if (likely(tcache != NULL)) {
if (likely(slab)) {
assert(sz_can_use_slab(size));
return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
} else if (likely(tcache_can_cache_large(tcache, ind))) {
return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
}
/* (size > tcache_max) case falls through. */
}
return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
}
JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(tsdn_t *tsdn, const void *ptr) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
@ -261,244 +240,6 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
return edata_usize_get(full_alloc_ctx.edata);
}
static inline void
arena_dalloc_large_no_tcache(
tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
/*
* szind is still needed in this function mainly becuase
* szind < SC_NBINS determines not only if this is a small alloc,
* but also if szind is valid (an inactive extent would have
* szind == SC_NSIZES).
*/
if (config_prof && unlikely(szind < SC_NBINS)) {
arena_dalloc_promoted(tsdn, ptr, NULL, true);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
static inline void
arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
assert(ptr != NULL);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
size_t usize, bool slow_path) {
assert(!tsdn_null(tsdn) && tcache != NULL);
bool is_sample_promoted = config_prof && szind < SC_NBINS;
if (unlikely(is_sample_promoted)) {
arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
} else {
if (tcache_can_cache_large(tcache, szind)) {
tcache_dalloc_large(
tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
}
JEMALLOC_ALWAYS_INLINE bool
arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
if (!config_debug) {
return false;
}
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
szind_t binind = edata_szind_get(edata);
div_info_t div_info = arena_binind_div_info[binind];
/*
* Calls the internal function bin_slab_regind_impl because the
* safety check does not require a lock.
*/
size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
slab_data_t *slab_data = edata_slab_data_get(edata);
const bin_info_t *bin_info = &bin_infos[binind];
assert(edata_nfree_get(edata) < bin_info->nregs);
if (unlikely(!bitmap_get(
slab_data->bitmap, &bin_info->bitmap_info, regind))) {
safety_check_fail(
"Invalid deallocation detected: the pointer being freed (%p) not "
"currently active, possibly caused by double free bugs.\n",
ptr);
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE void
arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
if (unlikely(tcache == NULL)) {
arena_dalloc_no_tcache(tsdn, ptr);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (caller_alloc_ctx != NULL) {
alloc_ctx = *caller_alloc_ctx;
} else {
util_assume(tsdn != NULL);
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
}
}
static inline void
arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
emap_alloc_ctx_t alloc_ctx;
if (!config_prof || !opt_prof) {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
szind_t szind = sz_size2index(size);
emap_alloc_ctx_init(
&alloc_ctx, szind, (szind < SC_NBINS), size);
}
if ((config_prof && opt_prof) || config_debug) {
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert((config_prof && opt_prof)
|| alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
}
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
if (unlikely(tcache == NULL)) {
arena_sdalloc_no_tcache(tsdn, ptr, size);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (config_prof && opt_prof) {
if (caller_alloc_ctx == NULL) {
/* Uncommon case and should be a static check. */
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
} else {
alloc_ctx = *caller_alloc_ctx;
}
} else {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
alloc_ctx.szind = sz_size2index(size);
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
emap_alloc_ctx_init(
&alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
sz_s2u(size), slow_path);
}
}
static inline void
arena_cache_oblivious_randomize(
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {

2
include/jemalloc/internal/jemalloc_internal_includes.h
View file

@ -61,6 +61,7 @@
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/large_externs.h"
#include "jemalloc/internal/tcache_externs.h"
#include "jemalloc/internal/malloc_dispatch.h"
#include "jemalloc/internal/prof_externs.h"
#include "jemalloc/internal/background_thread_externs.h"
@ -77,6 +78,7 @@
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
#include "jemalloc/internal/tcache_inlines.h"
#include "jemalloc/internal/arena_inlines_b.h"
#include "jemalloc/internal/malloc_dispatch_inlines.h"
#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
#include "jemalloc/internal/prof_inlines.h"
#include "jemalloc/internal/background_thread_inlines.h"

14
include/jemalloc/internal/jemalloc_internal_inlines_c.h
View file

@ -8,6 +8,7 @@
#include "jemalloc/internal/jemalloc_init.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/log.h"
#include "jemalloc/internal/malloc_dispatch_inlines.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/thread_event.h"
#include "jemalloc/internal/witness.h"
@ -67,7 +68,7 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
}
ret = arena_malloc(
ret = malloc_dispatch_malloc(
tsdn, arena, size, ind, zero, slab, tcache, slow_path);
if (config_stats && is_internal && likely(ret != NULL)) {
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@ -102,7 +103,8 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
ret = malloc_dispatch_palloc(
tsdn, arena, usize, alignment, zero, slab, tcache);
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
if (config_stats && is_internal && likely(ret != NULL)) {
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@ -156,7 +158,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
&& tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
assert(tcache == NULL);
}
arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
malloc_dispatch_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
}
JEMALLOC_ALWAYS_INLINE void
@ -169,7 +171,7 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
malloc_dispatch_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
}
JEMALLOC_ALWAYS_INLINE void *
@ -217,8 +219,8 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
zero, slab, tcache, arena);
}
return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
slab, tcache);
return malloc_dispatch_ralloc(
tsdn, arena, ptr, oldsize, size, alignment, zero, slab, tcache);
}
JEMALLOC_ALWAYS_INLINE void *

19
include/jemalloc/internal/malloc_dispatch.h Normal file
View file

@ -0,0 +1,19 @@
#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/tsd_types.h"
/* Forward decls; only used as pointer types below. */
typedef struct arena_s arena_t;
typedef struct tcache_s tcache_t;
void *malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, bool slab, tcache_t *tcache);
void malloc_dispatch_dalloc_promoted(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
void *malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr,
size_t oldsize, size_t size, size_t alignment, bool zero, bool slab,
tcache_t *tcache);
#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_H */

277
include/jemalloc/internal/malloc_dispatch_inlines.h Normal file
View file

@ -0,0 +1,277 @@
#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/arena_inlines_b.h"
#include "jemalloc/internal/bin_inlines.h"
#include "jemalloc/internal/div.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/large_externs.h"
#include "jemalloc/internal/malloc_dispatch.h"
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/tcache_inlines.h"
JEMALLOC_ALWAYS_INLINE void *
malloc_dispatch_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, bool slab, tcache_t *tcache, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
if (likely(tcache != NULL)) {
if (likely(slab)) {
assert(sz_can_use_slab(size));
return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
} else if (likely(tcache_can_cache_large(tcache, ind))) {
return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
size, ind, zero, slow_path);
}
/* (size > tcache_max) case falls through. */
}
return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
}
static inline void
malloc_dispatch_dalloc_large_no_tcache(
tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
/*
* szind both classifies small vs large and validates the extent --
* inactive extents have szind == SC_NSIZES.
*/
if (config_prof && unlikely(szind < SC_NBINS)) {
malloc_dispatch_dalloc_promoted(tsdn, ptr, NULL, true);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
static inline void
malloc_dispatch_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
assert(ptr != NULL);
emap_alloc_ctx_t alloc_ctx;
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
malloc_dispatch_dalloc_large_no_tcache(
tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
malloc_dispatch_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
szind_t szind, size_t usize, bool slow_path) {
assert(!tsdn_null(tsdn) && tcache != NULL);
bool is_sample_promoted = config_prof && szind < SC_NBINS;
if (unlikely(is_sample_promoted)) {
malloc_dispatch_dalloc_promoted(tsdn, ptr, tcache, slow_path);
} else {
if (tcache_can_cache_large(tcache, szind)) {
tcache_dalloc_large(
tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
} else {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
if (large_dalloc_safety_checks(edata, ptr, usize)) {
/* See the comment in isfree. */
return;
}
large_dalloc(tsdn, edata);
}
}
}
JEMALLOC_ALWAYS_INLINE bool
malloc_dispatch_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
if (!config_debug) {
return false;
}
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
szind_t binind = edata_szind_get(edata);
div_info_t div_info = arena_binind_div_info[binind];
/*
* Calls the internal function bin_slab_regind_impl because the
* safety check does not require a lock.
*/
size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
slab_data_t *slab_data = edata_slab_data_get(edata);
const bin_info_t *bin_info = &bin_infos[binind];
assert(edata_nfree_get(edata) < bin_info->nregs);
if (unlikely(!bitmap_get(
slab_data->bitmap, &bin_info->bitmap_info, regind))) {
safety_check_fail(
"Invalid deallocation detected: the pointer being freed (%p) not "
"currently active, possibly caused by double free bugs.\n",
ptr);
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE void
malloc_dispatch_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
if (unlikely(tcache == NULL)) {
malloc_dispatch_dalloc_no_tcache(tsdn, ptr);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (caller_alloc_ctx != NULL) {
alloc_ctx = *caller_alloc_ctx;
} else {
util_assume(tsdn != NULL);
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.szind < SC_NSIZES);
assert(alloc_ctx.slab == edata_slab_get(edata));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
}
}
static inline void
malloc_dispatch_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
emap_alloc_ctx_t alloc_ctx;
if (!config_prof || !opt_prof) {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
szind_t szind = sz_size2index(size);
emap_alloc_ctx_init(
&alloc_ctx, szind, (szind < SC_NBINS), size);
}
if ((config_prof && opt_prof) || config_debug) {
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert((config_prof && opt_prof)
|| alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
}
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
arena_dalloc_small(tsdn, ptr);
} else {
malloc_dispatch_dalloc_large_no_tcache(
tsdn, ptr, alloc_ctx.szind,
emap_alloc_ctx_usize_get(&alloc_ctx));
}
}
JEMALLOC_ALWAYS_INLINE void
malloc_dispatch_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
assert(!tsdn_null(tsdn) || tcache == NULL);
assert(ptr != NULL);
assert(size <= SC_LARGE_MAXCLASS);
if (unlikely(tcache == NULL)) {
malloc_dispatch_sdalloc_no_tcache(tsdn, ptr, size);
return;
}
emap_alloc_ctx_t alloc_ctx;
if (config_prof && opt_prof) {
if (caller_alloc_ctx == NULL) {
/* Uncommon case and should be a static check. */
emap_alloc_ctx_lookup(
tsdn, &arena_emap_global, ptr, &alloc_ctx);
assert(alloc_ctx.szind == sz_size2index(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
} else {
alloc_ctx = *caller_alloc_ctx;
}
} else {
/*
* There is no risk of being confused by a promoted sampled
* object, so base szind and slab on the given size.
*/
alloc_ctx.szind = sz_size2index(size);
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
}
if (config_debug) {
edata_t *edata = emap_edata_lookup(
tsdn, &arena_emap_global, ptr);
assert(alloc_ctx.szind == edata_szind_get(edata));
assert(alloc_ctx.slab == edata_slab_get(edata));
emap_alloc_ctx_init(
&alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
== edata_usize_get(edata));
}
if (likely(alloc_ctx.slab)) {
/* Small allocation. */
if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
return;
}
tcache_dalloc_small(
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
} else {
malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
sz_s2u(size), slow_path);
}
}
#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H */

1
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
View file

@ -73,6 +73,7 @@
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

3
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
View file

@ -94,6 +94,9 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>

1
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
View file

@ -73,6 +73,7 @@
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

3
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
View file

@ -94,6 +94,9 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>

1
msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
View file

@ -73,6 +73,7 @@
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

3
msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
View file

@ -94,6 +94,9 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>

1
msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
View file

@ -73,6 +73,7 @@
<ClCompile Include="..\..\..\..\src\jemalloc_init.c" />
<ClCompile Include="..\..\..\..\src\large.c" />
<ClCompile Include="..\..\..\..\src\log.c" />
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c" />
<ClCompile Include="..\..\..\..\src\malloc_io.c" />
<ClCompile Include="..\..\..\..\src\mutex.c" />
<ClCompile Include="..\..\..\..\src\nstime.c" />

3
msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
View file

@ -94,6 +94,9 @@
<ClCompile Include="..\..\..\..\src\log.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_dispatch.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\malloc_io.c">
<Filter>Source Files</Filter>
</ClCompile>

121
src/arena.c
View file

@ -694,11 +694,13 @@ arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize) {
assert(isalloc(tsdn, ptr) == usize);
}
static size_t
size_t
arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
cassert(config_prof);
assert(opt_prof);
assert(ptr != NULL);
size_t usize = isalloc(tsdn, ptr);
size_t usize = edata_usize_get(edata);
assert(isalloc(tsdn, ptr) == usize);
size_t bumped_usize = sz_sa2u(usize, PROF_SAMPLE_ALIGNMENT);
assert(bumped_usize <= SC_LARGE_MINCLASS
&& PAGE_CEILING(bumped_usize) == bumped_usize);
@ -710,17 +712,6 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
assert(isalloc(tsdn, ptr) == bumped_usize);
return bumped_usize;
}
static void
arena_dalloc_promoted_impl(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path, edata_t *edata) {
cassert(config_prof);
assert(opt_prof);
size_t usize = edata_usize_get(edata);
size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) {
/*
* Currently, we only do redzoning for small sampled
@ -728,21 +719,8 @@ arena_dalloc_promoted_impl(
*/
safety_check_verify_redzone(ptr, usize, bumped_usize);
}
szind_t bumped_ind = sz_size2index(bumped_usize);
if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
&& tcache_can_cache_large(tcache, bumped_ind)) {
tcache_dalloc_large(
tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
} else {
large_dalloc(tsdn, edata);
}
}
void
arena_dalloc_promoted(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
arena_dalloc_promoted_impl(tsdn, ptr, tcache, slow_path, edata);
return bumped_usize;
}
void
@ -784,8 +762,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
prof_free(tsd, ptr, usize, &alloc_ctx);
}
if (config_prof && opt_prof && alloc_ctx.szind < SC_NBINS) {
arena_dalloc_promoted_impl(tsd_tsdn(tsd), ptr,
/* tcache */ NULL, /* slow_path */ true, edata);
arena_prof_demote(tsd_tsdn(tsd), edata, ptr);
large_dalloc(tsd_tsdn(tsd), edata);
} else {
large_dalloc(tsd_tsdn(tsd), edata);
}
@ -1154,33 +1132,6 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
}
}
void *
arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
bool zero, bool slab, tcache_t *tcache) {
if (slab) {
assert(sz_can_use_slab(usize));
/* Small; alignment doesn't require special slab placement. */
/* usize should be a result of sz_sa2u() */
assert((usize & (alignment - 1)) == 0);
/*
* Small usize can't come from an alignment larger than a page.
*/
assert(alignment <= PAGE);
return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
zero, slab, tcache, true);
} else {
if (likely(alignment <= CACHELINE)) {
return large_malloc(tsdn, arena, usize, zero);
} else {
return large_palloc(
tsdn, arena, usize, alignment, zero);
}
}
}
static void
arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) {
szind_t binind = edata_szind_get(edata);
@ -1607,64 +1558,6 @@ done:
return ret;
}
static void *
arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, bool slab, tcache_t *tcache) {
if (alignment == 0) {
return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
zero, slab, tcache, true);
}
usize = sz_sa2u(usize, alignment);
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
return NULL;
}
return ipalloct_explicit_slab(
tsdn, usize, alignment, zero, slab, tcache, arena);
}
void *
arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
return NULL;
}
if (likely(slab)) {
assert(sz_can_use_slab(usize));
/* Try to avoid moving the allocation. */
UNUSED size_t newsize;
if (!arena_ralloc_no_move(
tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
return ptr;
}
}
if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
tcache);
}
/*
* size and oldsize are different enough that we need to move the
* object. In that case, fall back to allocating new space and copying.
*/
void *ret = arena_ralloc_move_helper(
tsdn, arena, usize, alignment, zero, slab, tcache);
if (ret == NULL) {
return NULL;
}
/*
* Junk/zero-filling were already done by
* ipalloc()/arena_malloc().
*/
size_t copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
return ret;
}
ehooks_t *
arena_get_ehooks(const arena_t *arena) {
return base_ehooks_get(arena->base);

109
src/malloc_dispatch.c Normal file
View file

@ -0,0 +1,109 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/assert.h"
#include "jemalloc/internal/emap.h"
/******************************************************************************/
void
malloc_dispatch_dalloc_promoted(
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
cassert(config_prof);
assert(opt_prof);
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
szind_t bumped_ind = sz_size2index(bumped_usize);
if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
&& tcache_can_cache_large(tcache, bumped_ind)) {
tcache_dalloc_large(
tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
} else {
large_dalloc(tsdn, edata);
}
}
void *
malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, bool slab, tcache_t *tcache) {
if (slab) {
assert(sz_can_use_slab(usize));
/* Small; alignment doesn't require special slab placement. */
/* usize should be a result of sz_sa2u() */
assert((usize & (alignment - 1)) == 0);
/*
* Small usize can't come from an alignment larger than a page.
*/
assert(alignment <= PAGE);
return malloc_dispatch_malloc(tsdn, arena, usize,
sz_size2index(usize), zero, slab, tcache, true);
} else {
if (likely(alignment <= CACHELINE)) {
return large_malloc(tsdn, arena, usize, zero);
} else {
return large_palloc(
tsdn, arena, usize, alignment, zero);
}
}
}
static void *
malloc_dispatch_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero, bool slab, tcache_t *tcache) {
if (alignment == 0) {
return malloc_dispatch_malloc(tsdn, arena, usize,
sz_size2index(usize), zero, slab, tcache, true);
}
usize = sz_sa2u(usize, alignment);
if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
return NULL;
}
return ipalloct_explicit_slab(
tsdn, usize, alignment, zero, slab, tcache, arena);
}
void *
malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
return NULL;
}
if (likely(slab)) {
assert(sz_can_use_slab(usize));
/* Try to avoid moving the allocation. */
UNUSED size_t newsize;
if (!arena_ralloc_no_move(
tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
return ptr;
}
}
if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
tcache);
}
/*
* size and oldsize are different enough that we need to move the
* object. In that case, fall back to allocating new space and copying.
*/
void *ret = malloc_dispatch_ralloc_move_helper(
tsdn, arena, usize, alignment, zero, slab, tcache);
if (ret == NULL) {
return NULL;
}
/*
* Junk/zero-filling were already done by ipalloc() / dispatch alloc.
*/
size_t copysize = (usize < oldsize) ? usize : oldsize;
memcpy(ret, ptr, copysize);
isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
return ret;
}

208
test/unit/malloc_dispatch.c Normal file
View file

@ -0,0 +1,208 @@
#include "test/jemalloc_test.h"
/*
* Targeted coverage for the malloc_dispatch routing layer. Integration
* tests exercise the public API end-to-end but don't assert which branch
* was taken for a given (size, tcache) input. These tests make the
* routing observable through per-arena stats counters:
*
* 1. small + tcache != NULL -> tcache_alloc_small
* 2. small + tcache == NULL -> arena_malloc_hard + arena_dalloc_small
* 3. large <= tcache_max + tcache -> tcache_alloc_large
* 4. large > tcache_max + tcache -> arena_malloc_hard + large_dalloc
*/
static unsigned
create_fresh_arena(void) {
unsigned arena_ind;
size_t sz = sizeof(arena_ind);
expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
0, "arenas.create failed");
return arena_ind;
}
static void
refresh_stats(void) {
uint64_t epoch = 1;
expect_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0,
"epoch refresh failed");
}
static uint64_t
read_u64_mallctl(const char *cmd) {
uint64_t v;
size_t sz = sizeof(v);
expect_d_eq(mallctl(cmd, (void *)&v, &sz, NULL, 0), 0,
"mallctl read failed");
return v;
}
static uint64_t
arena_bin0_nmalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.bins.0.nmalloc", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_bin0_ndalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.bins.0.ndalloc", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_bin0_nfills(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.bins.0.nfills", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_large_nmalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.large.nmalloc", arena_ind);
return read_u64_mallctl(cmd);
}
static uint64_t
arena_large_ndalloc(unsigned arena_ind) {
char cmd[128];
(void)snprintf(
cmd, sizeof(cmd), "stats.arenas.%u.large.ndalloc", arena_ind);
return read_u64_mallctl(cmd);
}
/*
* Branch 2: small alloc/dalloc with MALLOCX_TCACHE_NONE must increment the
* arena bin counters immediately (no tcache to absorb the call).
*/
TEST_BEGIN(test_dispatch_small_no_tcache) {
test_skip_if(!config_stats);
test_skip_if(opt_prof);
unsigned arena_ind = create_fresh_arena();
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
size_t sz = bin_infos[0].reg_size;
refresh_stats();
uint64_t nmalloc_before = arena_bin0_nmalloc(arena_ind);
uint64_t ndalloc_before = arena_bin0_ndalloc(arena_ind);
uint64_t nfills_before = arena_bin0_nfills(arena_ind);
void *p = mallocx(sz, flags);
expect_ptr_not_null(p, "mallocx failed");
dallocx(p, flags);
refresh_stats();
expect_u64_eq(arena_bin0_nmalloc(arena_ind), nmalloc_before + 1,
"small no-tcache alloc must increment arena bin nmalloc");
expect_u64_eq(arena_bin0_ndalloc(arena_ind), ndalloc_before + 1,
"small no-tcache dalloc must increment arena bin ndalloc");
expect_u64_eq(arena_bin0_nfills(arena_ind), nfills_before,
"no-tcache path must not trigger any tcache fill");
}
TEST_END
/*
* Branch 1: small alloc with tcache must NOT increment the arena bin nmalloc
* counter directly (the tcache absorbs the alloc); a refill (nfills > 0) is
* the routing observable instead. Bound the test to a single tcache slot to
* make the refill predictable.
*/
TEST_BEGIN(test_dispatch_small_with_tcache) {
test_skip_if(!config_stats);
test_skip_if(!opt_tcache);
test_skip_if(opt_prof);
unsigned arena_ind = create_fresh_arena();
int flags = MALLOCX_ARENA(arena_ind);
size_t sz = bin_infos[0].reg_size;
/* Flush any per-thread tcache so this arena's tcache slot starts cold. */
expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0,
"thread.tcache.flush failed");
refresh_stats();
uint64_t nfills_before = arena_bin0_nfills(arena_ind);
void *p = mallocx(sz, flags);
expect_ptr_not_null(p, "mallocx failed");
refresh_stats();
expect_u64_gt(arena_bin0_nfills(arena_ind), nfills_before,
"small alloc with tcache must trigger a tcache fill");
dallocx(p, flags);
}
TEST_END
/*
* Branches 3 & 4: large alloc routing pivots on tcache_max. Use a fresh
* thread.tcache.max so we can compute a size that is just above it; that size
* must be routed through arena_malloc_hard (and not the tcache). Allocations
* below tcache_max routed through the same call site must not touch the arena
* large counters on the malloc/free pair (they ride the tcache).
*/
TEST_BEGIN(test_dispatch_large_routes_on_tcache_max) {
test_skip_if(!config_stats);
test_skip_if(!opt_tcache);
test_skip_if(opt_prof);
test_skip_if(san_uaf_detection_enabled());
unsigned arena_ind = create_fresh_arena();
expect_d_eq(
mallctl("thread.arena", NULL, NULL, &arena_ind, sizeof(arena_ind)),
0, "thread.arena bind failed");
/* Pin tcache_max to a known small-large boundary. */
size_t small_large;
size_t sz = sizeof(small_large);
expect_d_eq(mallctl("arenas.lextent.0.size", (void *)&small_large, &sz,
NULL, 0),
0, "arenas.lextent.0.size lookup failed");
expect_d_eq(mallctl("thread.tcache.max", NULL, NULL, &small_large,
sizeof(small_large)),
0, "thread.tcache.max set failed");
expect_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), 0,
"thread.tcache.flush failed");
int flags = MALLOCX_ARENA(arena_ind);
/* Above tcache_max: must hit arena_malloc_hard + large_dalloc. */
size_t too_big;
expect_d_eq(mallctl("arenas.lextent.1.size", (void *)&too_big, &sz, NULL,
0),
0, "arenas.lextent.1.size lookup failed");
expect_zu_gt(too_big, small_large,
"lextent.1 must exceed tcache_max boundary");
refresh_stats();
uint64_t nmalloc_before = arena_large_nmalloc(arena_ind);
uint64_t ndalloc_before = arena_large_ndalloc(arena_ind);
void *big = mallocx(too_big, flags);
expect_ptr_not_null(big, "mallocx failed");
refresh_stats();
expect_u64_eq(arena_large_nmalloc(arena_ind), nmalloc_before + 1,
"large alloc above tcache_max must bypass tcache "
"and increment arena large nmalloc");
dallocx(big, flags);
refresh_stats();
expect_u64_eq(arena_large_ndalloc(arena_ind), ndalloc_before + 1,
"large dalloc above tcache_max must bypass tcache "
"and increment arena large ndalloc");
}
TEST_END
int
main(void) {
return test(test_dispatch_small_no_tcache,
test_dispatch_small_with_tcache,
test_dispatch_large_routes_on_tcache_max);
}