mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-06-02 10:14:15 +03:00
Move malloc routing into new malloc_dispatch module
Pull the tcache-aware allocation routing helpers out of arena into a layer that sits directly below the public malloc interface: arena_malloc -> malloc_dispatch_malloc arena_palloc -> malloc_dispatch_palloc arena_ralloc -> malloc_dispatch_ralloc arena_dalloc* -> malloc_dispatch_dalloc* arena_sdalloc* -> malloc_dispatch_sdalloc* arena_dalloc_promoted -> malloc_dispatch_dalloc_promoted These helpers decide whether to route through tcache or fall through to arena/large fast paths. They are now owned by malloc_dispatch_inlines.h + src/malloc_dispatch.c, and the only consumers are the public-front-end wrappers in jemalloc_internal_inlines_c.h. arena keeps a narrower arena_prof_demote() helper for the sampled allocation demotion + redzone verification it used to perform inline. arena_inlines_b.h no longer includes tcache_inlines.h -- the symbol level arena <-> tcache cycle is gone (it's now in malloc_dispatch).
This commit is contained in:
Slobodan Predolac
Slobodan Predolac
parent
1cc0e4a838
commit
e2c0f07a7e
17 changed files with 442 additions and 385 deletions
|
|
@ -70,12 +70,9 @@ cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
|
|||
|
||||
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
|
||||
bool zero, bool slab);
|
||||
void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
||||
bool zero, bool slab, tcache_t *tcache);
|
||||
void arena_prof_promote(
|
||||
tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
|
||||
void arena_dalloc_promoted(
|
||||
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
|
||||
size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
|
||||
void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
|
||||
|
||||
void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
|
||||
|
|
@ -84,8 +81,6 @@ void arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
|
|||
arena_t *stats_arena, cache_bin_stats_t merge_stats);
|
||||
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
|
||||
size_t extra, bool zero, size_t *newsize);
|
||||
void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
|
||||
size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache);
|
||||
dss_prec_t arena_dss_prec_get(const arena_t *arena);
|
||||
ehooks_t *arena_get_ehooks(const arena_t *arena);
|
||||
extent_hooks_t *arena_set_extent_hooks(
|
||||
|
|
|
|||
|
|
@ -17,7 +17,6 @@
|
|||
#include "jemalloc/internal/safety_check.h"
|
||||
#include "jemalloc/internal/sc.h"
|
||||
#include "jemalloc/internal/sz.h"
|
||||
#include "jemalloc/internal/tcache_inlines.h"
|
||||
#include "jemalloc/internal/ticker.h"
|
||||
|
||||
static inline arena_t *
|
||||
|
|
@ -193,26 +192,6 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
|
|||
arena_decay_ticks(tsdn, arena, 1);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
|
||||
bool slab, tcache_t *tcache, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) || tcache == NULL);
|
||||
|
||||
if (likely(tcache != NULL)) {
|
||||
if (likely(slab)) {
|
||||
assert(sz_can_use_slab(size));
|
||||
return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
|
||||
size, ind, zero, slow_path);
|
||||
} else if (likely(tcache_can_cache_large(tcache, ind))) {
|
||||
return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
|
||||
size, ind, zero, slow_path);
|
||||
}
|
||||
/* (size > tcache_max) case falls through. */
|
||||
}
|
||||
|
||||
return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE arena_t *
|
||||
arena_aalloc(tsdn_t *tsdn, const void *ptr) {
|
||||
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
|
||||
|
|
@ -261,244 +240,6 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
|
|||
return edata_usize_get(full_alloc_ctx.edata);
|
||||
}
|
||||
|
||||
static inline void
|
||||
arena_dalloc_large_no_tcache(
|
||||
tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
|
||||
/*
|
||||
* szind is still needed in this function mainly becuase
|
||||
* szind < SC_NBINS determines not only if this is a small alloc,
|
||||
* but also if szind is valid (an inactive extent would have
|
||||
* szind == SC_NSIZES).
|
||||
*/
|
||||
if (config_prof && unlikely(szind < SC_NBINS)) {
|
||||
arena_dalloc_promoted(tsdn, ptr, NULL, true);
|
||||
} else {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
if (large_dalloc_safety_checks(edata, ptr, usize)) {
|
||||
/* See the comment in isfree. */
|
||||
return;
|
||||
}
|
||||
large_dalloc(tsdn, edata);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
|
||||
assert(ptr != NULL);
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.szind < SC_NSIZES);
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
|
||||
== edata_usize_get(edata));
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
arena_dalloc_small(tsdn, ptr);
|
||||
} else {
|
||||
arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
|
||||
emap_alloc_ctx_usize_get(&alloc_ctx));
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
|
||||
size_t usize, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) && tcache != NULL);
|
||||
bool is_sample_promoted = config_prof && szind < SC_NBINS;
|
||||
if (unlikely(is_sample_promoted)) {
|
||||
arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
|
||||
} else {
|
||||
if (tcache_can_cache_large(tcache, szind)) {
|
||||
tcache_dalloc_large(
|
||||
tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
|
||||
} else {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
if (large_dalloc_safety_checks(edata, ptr, usize)) {
|
||||
/* See the comment in isfree. */
|
||||
return;
|
||||
}
|
||||
large_dalloc(tsdn, edata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
|
||||
if (!config_debug) {
|
||||
return false;
|
||||
}
|
||||
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
|
||||
szind_t binind = edata_szind_get(edata);
|
||||
div_info_t div_info = arena_binind_div_info[binind];
|
||||
/*
|
||||
* Calls the internal function bin_slab_regind_impl because the
|
||||
* safety check does not require a lock.
|
||||
*/
|
||||
size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
|
||||
slab_data_t *slab_data = edata_slab_data_get(edata);
|
||||
const bin_info_t *bin_info = &bin_infos[binind];
|
||||
assert(edata_nfree_get(edata) < bin_info->nregs);
|
||||
if (unlikely(!bitmap_get(
|
||||
slab_data->bitmap, &bin_info->bitmap_info, regind))) {
|
||||
safety_check_fail(
|
||||
"Invalid deallocation detected: the pointer being freed (%p) not "
|
||||
"currently active, possibly caused by double free bugs.\n",
|
||||
ptr);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
|
||||
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) || tcache == NULL);
|
||||
assert(ptr != NULL);
|
||||
|
||||
if (unlikely(tcache == NULL)) {
|
||||
arena_dalloc_no_tcache(tsdn, ptr);
|
||||
return;
|
||||
}
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
if (caller_alloc_ctx != NULL) {
|
||||
alloc_ctx = *caller_alloc_ctx;
|
||||
} else {
|
||||
util_assume(tsdn != NULL);
|
||||
emap_alloc_ctx_lookup(
|
||||
tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
}
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.szind < SC_NSIZES);
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
|
||||
== edata_usize_get(edata));
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
|
||||
return;
|
||||
}
|
||||
tcache_dalloc_small(
|
||||
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
|
||||
} else {
|
||||
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
|
||||
emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
|
||||
assert(ptr != NULL);
|
||||
assert(size <= SC_LARGE_MAXCLASS);
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
if (!config_prof || !opt_prof) {
|
||||
/*
|
||||
* There is no risk of being confused by a promoted sampled
|
||||
* object, so base szind and slab on the given size.
|
||||
*/
|
||||
szind_t szind = sz_size2index(size);
|
||||
emap_alloc_ctx_init(
|
||||
&alloc_ctx, szind, (szind < SC_NBINS), size);
|
||||
}
|
||||
|
||||
if ((config_prof && opt_prof) || config_debug) {
|
||||
emap_alloc_ctx_lookup(
|
||||
tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
|
||||
assert(alloc_ctx.szind == sz_size2index(size));
|
||||
assert((config_prof && opt_prof)
|
||||
|| alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
}
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
arena_dalloc_small(tsdn, ptr);
|
||||
} else {
|
||||
arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
|
||||
emap_alloc_ctx_usize_get(&alloc_ctx));
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
|
||||
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) || tcache == NULL);
|
||||
assert(ptr != NULL);
|
||||
assert(size <= SC_LARGE_MAXCLASS);
|
||||
|
||||
if (unlikely(tcache == NULL)) {
|
||||
arena_sdalloc_no_tcache(tsdn, ptr, size);
|
||||
return;
|
||||
}
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
if (config_prof && opt_prof) {
|
||||
if (caller_alloc_ctx == NULL) {
|
||||
/* Uncommon case and should be a static check. */
|
||||
emap_alloc_ctx_lookup(
|
||||
tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
assert(alloc_ctx.szind == sz_size2index(size));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
|
||||
} else {
|
||||
alloc_ctx = *caller_alloc_ctx;
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
* There is no risk of being confused by a promoted sampled
|
||||
* object, so base szind and slab on the given size.
|
||||
*/
|
||||
alloc_ctx.szind = sz_size2index(size);
|
||||
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
|
||||
}
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
emap_alloc_ctx_init(
|
||||
&alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
|
||||
== edata_usize_get(edata));
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
|
||||
return;
|
||||
}
|
||||
tcache_dalloc_small(
|
||||
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
|
||||
} else {
|
||||
arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
|
||||
sz_s2u(size), slow_path);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
arena_cache_oblivious_randomize(
|
||||
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@
|
|||
#include "jemalloc/internal/arena_externs.h"
|
||||
#include "jemalloc/internal/large_externs.h"
|
||||
#include "jemalloc/internal/tcache_externs.h"
|
||||
#include "jemalloc/internal/malloc_dispatch_externs.h"
|
||||
#include "jemalloc/internal/prof_externs.h"
|
||||
#include "jemalloc/internal/background_thread_externs.h"
|
||||
|
||||
|
|
@ -77,6 +78,7 @@
|
|||
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
|
||||
#include "jemalloc/internal/tcache_inlines.h"
|
||||
#include "jemalloc/internal/arena_inlines_b.h"
|
||||
#include "jemalloc/internal/malloc_dispatch_inlines.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
|
||||
#include "jemalloc/internal/prof_inlines.h"
|
||||
#include "jemalloc/internal/background_thread_inlines.h"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
#include "jemalloc/internal/jemalloc_init.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_types.h"
|
||||
#include "jemalloc/internal/log.h"
|
||||
#include "jemalloc/internal/malloc_dispatch_inlines.h"
|
||||
#include "jemalloc/internal/sz.h"
|
||||
#include "jemalloc/internal/thread_event.h"
|
||||
#include "jemalloc/internal/witness.h"
|
||||
|
|
@ -67,7 +68,7 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
|
|||
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
|
||||
}
|
||||
|
||||
ret = arena_malloc(
|
||||
ret = malloc_dispatch_malloc(
|
||||
tsdn, arena, size, ind, zero, slab, tcache, slow_path);
|
||||
if (config_stats && is_internal && likely(ret != NULL)) {
|
||||
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
|
||||
|
|
@ -102,7 +103,8 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
|
|||
witness_assert_depth_to_rank(
|
||||
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
|
||||
|
||||
ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
|
||||
ret = malloc_dispatch_palloc(
|
||||
tsdn, arena, usize, alignment, zero, slab, tcache);
|
||||
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
|
||||
if (config_stats && is_internal && likely(ret != NULL)) {
|
||||
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
|
||||
|
|
@ -156,7 +158,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
|
|||
&& tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
|
||||
assert(tcache == NULL);
|
||||
}
|
||||
arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
|
||||
malloc_dispatch_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
|
|
@ -169,7 +171,7 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
|
|||
emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
|
||||
witness_assert_depth_to_rank(
|
||||
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
|
||||
arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
|
||||
malloc_dispatch_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
|
|
@ -217,8 +219,8 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
|
|||
zero, slab, tcache, arena);
|
||||
}
|
||||
|
||||
return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
|
||||
slab, tcache);
|
||||
return malloc_dispatch_ralloc(
|
||||
tsdn, arena, ptr, oldsize, size, alignment, zero, slab, tcache);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
|
|
|
|||
19
include/jemalloc/internal/malloc_dispatch_externs.h
Normal file
19
include/jemalloc/internal/malloc_dispatch_externs.h
Normal file
|
|
@ -0,0 +1,19 @@
|
|||
#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_EXTERNS_H
|
||||
#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_EXTERNS_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/tsd_types.h"
|
||||
|
||||
/* Forward decls; only used as pointer types below. */
|
||||
typedef struct arena_s arena_t;
|
||||
typedef struct tcache_s tcache_t;
|
||||
|
||||
void *malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
||||
size_t alignment, bool zero, bool slab, tcache_t *tcache);
|
||||
void malloc_dispatch_dalloc_promoted(
|
||||
tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
|
||||
void *malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr,
|
||||
size_t oldsize, size_t size, size_t alignment, bool zero, bool slab,
|
||||
tcache_t *tcache);
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_EXTERNS_H */
|
||||
279
include/jemalloc/internal/malloc_dispatch_inlines.h
Normal file
279
include/jemalloc/internal/malloc_dispatch_inlines.h
Normal file
|
|
@ -0,0 +1,279 @@
|
|||
#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
|
||||
#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/arena_externs.h"
|
||||
#include "jemalloc/internal/arena_inlines_b.h"
|
||||
#include "jemalloc/internal/bin_inlines.h"
|
||||
#include "jemalloc/internal/div.h"
|
||||
#include "jemalloc/internal/emap.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_types.h"
|
||||
#include "jemalloc/internal/large_externs.h"
|
||||
#include "jemalloc/internal/malloc_dispatch_externs.h"
|
||||
#include "jemalloc/internal/safety_check.h"
|
||||
#include "jemalloc/internal/sc.h"
|
||||
#include "jemalloc/internal/sz.h"
|
||||
#include "jemalloc/internal/tcache_inlines.h"
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
malloc_dispatch_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
|
||||
bool zero, bool slab, tcache_t *tcache, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) || tcache == NULL);
|
||||
|
||||
if (likely(tcache != NULL)) {
|
||||
if (likely(slab)) {
|
||||
assert(sz_can_use_slab(size));
|
||||
return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
|
||||
size, ind, zero, slow_path);
|
||||
} else if (likely(tcache_can_cache_large(tcache, ind))) {
|
||||
return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
|
||||
size, ind, zero, slow_path);
|
||||
}
|
||||
/* (size > tcache_max) case falls through. */
|
||||
}
|
||||
|
||||
return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
|
||||
}
|
||||
|
||||
static inline void
|
||||
malloc_dispatch_dalloc_large_no_tcache(
|
||||
tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
|
||||
/*
|
||||
* szind is still needed in this function mainly because
|
||||
* szind < SC_NBINS determines not only if this is a small alloc,
|
||||
* but also if szind is valid (an inactive extent would have
|
||||
* szind == SC_NSIZES).
|
||||
*/
|
||||
if (config_prof && unlikely(szind < SC_NBINS)) {
|
||||
malloc_dispatch_dalloc_promoted(tsdn, ptr, NULL, true);
|
||||
} else {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
if (large_dalloc_safety_checks(edata, ptr, usize)) {
|
||||
/* See the comment in isfree. */
|
||||
return;
|
||||
}
|
||||
large_dalloc(tsdn, edata);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
malloc_dispatch_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
|
||||
assert(ptr != NULL);
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.szind < SC_NSIZES);
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
|
||||
== edata_usize_get(edata));
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
arena_dalloc_small(tsdn, ptr);
|
||||
} else {
|
||||
malloc_dispatch_dalloc_large_no_tcache(
|
||||
tsdn, ptr, alloc_ctx.szind,
|
||||
emap_alloc_ctx_usize_get(&alloc_ctx));
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
malloc_dispatch_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
|
||||
szind_t szind, size_t usize, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) && tcache != NULL);
|
||||
bool is_sample_promoted = config_prof && szind < SC_NBINS;
|
||||
if (unlikely(is_sample_promoted)) {
|
||||
malloc_dispatch_dalloc_promoted(tsdn, ptr, tcache, slow_path);
|
||||
} else {
|
||||
if (tcache_can_cache_large(tcache, szind)) {
|
||||
tcache_dalloc_large(
|
||||
tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
|
||||
} else {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
if (large_dalloc_safety_checks(edata, ptr, usize)) {
|
||||
/* See the comment in isfree. */
|
||||
return;
|
||||
}
|
||||
large_dalloc(tsdn, edata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
malloc_dispatch_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
|
||||
if (!config_debug) {
|
||||
return false;
|
||||
}
|
||||
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
|
||||
szind_t binind = edata_szind_get(edata);
|
||||
div_info_t div_info = arena_binind_div_info[binind];
|
||||
/*
|
||||
* Calls the internal function bin_slab_regind_impl because the
|
||||
* safety check does not require a lock.
|
||||
*/
|
||||
size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
|
||||
slab_data_t *slab_data = edata_slab_data_get(edata);
|
||||
const bin_info_t *bin_info = &bin_infos[binind];
|
||||
assert(edata_nfree_get(edata) < bin_info->nregs);
|
||||
if (unlikely(!bitmap_get(
|
||||
slab_data->bitmap, &bin_info->bitmap_info, regind))) {
|
||||
safety_check_fail(
|
||||
"Invalid deallocation detected: the pointer being freed (%p) not "
|
||||
"currently active, possibly caused by double free bugs.\n",
|
||||
ptr);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
malloc_dispatch_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
|
||||
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) || tcache == NULL);
|
||||
assert(ptr != NULL);
|
||||
|
||||
if (unlikely(tcache == NULL)) {
|
||||
malloc_dispatch_dalloc_no_tcache(tsdn, ptr);
|
||||
return;
|
||||
}
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
if (caller_alloc_ctx != NULL) {
|
||||
alloc_ctx = *caller_alloc_ctx;
|
||||
} else {
|
||||
util_assume(tsdn != NULL);
|
||||
emap_alloc_ctx_lookup(
|
||||
tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
}
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.szind < SC_NSIZES);
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
|
||||
== edata_usize_get(edata));
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
|
||||
return;
|
||||
}
|
||||
tcache_dalloc_small(
|
||||
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
|
||||
} else {
|
||||
malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
|
||||
emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
malloc_dispatch_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
|
||||
assert(ptr != NULL);
|
||||
assert(size <= SC_LARGE_MAXCLASS);
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
if (!config_prof || !opt_prof) {
|
||||
/*
|
||||
* There is no risk of being confused by a promoted sampled
|
||||
* object, so base szind and slab on the given size.
|
||||
*/
|
||||
szind_t szind = sz_size2index(size);
|
||||
emap_alloc_ctx_init(
|
||||
&alloc_ctx, szind, (szind < SC_NBINS), size);
|
||||
}
|
||||
|
||||
if ((config_prof && opt_prof) || config_debug) {
|
||||
emap_alloc_ctx_lookup(
|
||||
tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
|
||||
assert(alloc_ctx.szind == sz_size2index(size));
|
||||
assert((config_prof && opt_prof)
|
||||
|| alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
}
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
arena_dalloc_small(tsdn, ptr);
|
||||
} else {
|
||||
malloc_dispatch_dalloc_large_no_tcache(
|
||||
tsdn, ptr, alloc_ctx.szind,
|
||||
emap_alloc_ctx_usize_get(&alloc_ctx));
|
||||
}
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void
|
||||
malloc_dispatch_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
|
||||
emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
|
||||
assert(!tsdn_null(tsdn) || tcache == NULL);
|
||||
assert(ptr != NULL);
|
||||
assert(size <= SC_LARGE_MAXCLASS);
|
||||
|
||||
if (unlikely(tcache == NULL)) {
|
||||
malloc_dispatch_sdalloc_no_tcache(tsdn, ptr, size);
|
||||
return;
|
||||
}
|
||||
|
||||
emap_alloc_ctx_t alloc_ctx;
|
||||
if (config_prof && opt_prof) {
|
||||
if (caller_alloc_ctx == NULL) {
|
||||
/* Uncommon case and should be a static check. */
|
||||
emap_alloc_ctx_lookup(
|
||||
tsdn, &arena_emap_global, ptr, &alloc_ctx);
|
||||
assert(alloc_ctx.szind == sz_size2index(size));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
|
||||
} else {
|
||||
alloc_ctx = *caller_alloc_ctx;
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
* There is no risk of being confused by a promoted sampled
|
||||
* object, so base szind and slab on the given size.
|
||||
*/
|
||||
alloc_ctx.szind = sz_size2index(size);
|
||||
alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
|
||||
}
|
||||
|
||||
if (config_debug) {
|
||||
edata_t *edata = emap_edata_lookup(
|
||||
tsdn, &arena_emap_global, ptr);
|
||||
assert(alloc_ctx.szind == edata_szind_get(edata));
|
||||
assert(alloc_ctx.slab == edata_slab_get(edata));
|
||||
emap_alloc_ctx_init(
|
||||
&alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
|
||||
assert(emap_alloc_ctx_usize_get(&alloc_ctx)
|
||||
== edata_usize_get(edata));
|
||||
}
|
||||
|
||||
if (likely(alloc_ctx.slab)) {
|
||||
/* Small allocation. */
|
||||
if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
|
||||
return;
|
||||
}
|
||||
tcache_dalloc_small(
|
||||
tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
|
||||
} else {
|
||||
malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
|
||||
sz_s2u(size), slow_path);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H */
|
||||
Loading…
Add table
Add a link
Reference in a new issue