mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-05-18 19:06:25 +03:00
Use rtree-based chunk lookups rather than pointer bit twiddling.
Look up chunk metadata via the radix tree, rather than using CHUNK_ADDR2BASE(). Propagate pointer's containing extent. Minimize extent lookups by doing a single lookup (e.g. in free()) and propagating the pointer's extent into nearly all the functions that may need it.
This commit is contained in:
Jason Evans
parent
2d2b4e98c9
commit
db72272bef
14 changed files with 548 additions and 504 deletions
245
src/arena.c
245
src/arena.c
|
|
@ -45,10 +45,10 @@ unsigned nhclasses; /* Number of huge size classes. */
|
|||
|
||||
static void arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena,
|
||||
size_t ndirty_limit);
|
||||
static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run,
|
||||
bool dirty, bool cleaned, bool decommitted);
|
||||
static void arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
arena_run_t *run, bool dirty, bool cleaned, bool decommitted);
|
||||
static void arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin);
|
||||
arena_chunk_t *chunk, extent_t *extent, arena_run_t *run, arena_bin_t *bin);
|
||||
static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk,
|
||||
arena_run_t *run, arena_bin_t *bin);
|
||||
|
||||
|
|
@ -264,9 +264,9 @@ arena_run_reg_alloc(arena_run_t *run, const arena_bin_info_t *bin_info)
|
|||
}
|
||||
|
||||
JEMALLOC_INLINE_C void
|
||||
arena_run_reg_dalloc(arena_run_t *run, void *ptr)
|
||||
arena_run_reg_dalloc(arena_run_t *run, extent_t *extent, void *ptr)
|
||||
{
|
||||
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
||||
arena_chunk_t *chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
|
||||
size_t mapbits = arena_mapbits_get(chunk, pageind);
|
||||
szind_t binind = arena_ptr_small_binind_get(ptr, mapbits);
|
||||
|
|
@ -375,15 +375,15 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind,
|
|||
}
|
||||
|
||||
static bool
|
||||
arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,
|
||||
bool remove, bool zero)
|
||||
arena_run_split_large_helper(arena_t *arena, extent_t *extent, arena_run_t *run,
|
||||
size_t size, bool remove, bool zero)
|
||||
{
|
||||
arena_chunk_t *chunk;
|
||||
arena_chunk_map_misc_t *miscelm;
|
||||
size_t flag_dirty, flag_decommitted, run_ind, need_pages;
|
||||
size_t flag_unzeroed_mask;
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
||||
chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
miscelm = arena_run_to_miscelm(run);
|
||||
run_ind = arena_miscelm_to_pageind(miscelm);
|
||||
flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
|
||||
|
|
@ -439,22 +439,26 @@ arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size,
|
|||
}
|
||||
|
||||
static bool
|
||||
arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
|
||||
arena_run_split_large(arena_t *arena, extent_t *extent, arena_run_t *run,
|
||||
size_t size, bool zero)
|
||||
{
|
||||
|
||||
return (arena_run_split_large_helper(arena, run, size, true, zero));
|
||||
return (arena_run_split_large_helper(arena, extent, run, size, true,
|
||||
zero));
|
||||
}
|
||||
|
||||
static bool
|
||||
arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero)
|
||||
arena_run_init_large(arena_t *arena, extent_t *extent, arena_run_t *run,
|
||||
size_t size, bool zero)
|
||||
{
|
||||
|
||||
return (arena_run_split_large_helper(arena, run, size, false, zero));
|
||||
return (arena_run_split_large_helper(arena, extent, run, size, false,
|
||||
zero));
|
||||
}
|
||||
|
||||
static bool
|
||||
arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
|
||||
szind_t binind)
|
||||
arena_run_split_small(arena_t *arena, extent_t *extent, arena_run_t *run,
|
||||
size_t size, szind_t binind)
|
||||
{
|
||||
arena_chunk_t *chunk;
|
||||
arena_chunk_map_misc_t *miscelm;
|
||||
|
|
@ -462,7 +466,7 @@ arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size,
|
|||
|
||||
assert(binind != BININD_INVALID);
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
||||
chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
miscelm = arena_run_to_miscelm(run);
|
||||
run_ind = arena_miscelm_to_pageind(miscelm);
|
||||
flag_dirty = arena_mapbits_dirty_get(chunk, run_ind);
|
||||
|
|
@ -1037,7 +1041,7 @@ arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero)
|
|||
{
|
||||
arena_run_t *run = arena_run_first_best_fit(arena, s2u(size));
|
||||
if (run != NULL) {
|
||||
if (arena_run_split_large(arena, run, size, zero))
|
||||
if (arena_run_split_large(arena, iealloc(run), run, size, zero))
|
||||
run = NULL;
|
||||
}
|
||||
return (run);
|
||||
|
|
@ -1063,7 +1067,7 @@ arena_run_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t size, bool zero)
|
|||
chunk = arena_chunk_alloc(tsdn, arena);
|
||||
if (chunk != NULL) {
|
||||
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
|
||||
if (arena_run_split_large(arena, run, size, zero))
|
||||
if (arena_run_split_large(arena, iealloc(run), run, size, zero))
|
||||
run = NULL;
|
||||
return (run);
|
||||
}
|
||||
|
|
@ -1081,7 +1085,8 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, szind_t binind)
|
|||
{
|
||||
arena_run_t *run = arena_run_first_best_fit(arena, size);
|
||||
if (run != NULL) {
|
||||
if (arena_run_split_small(arena, run, size, binind))
|
||||
if (arena_run_split_small(arena, iealloc(run), run, size,
|
||||
binind))
|
||||
run = NULL;
|
||||
}
|
||||
return (run);
|
||||
|
|
@ -1108,7 +1113,8 @@ arena_run_alloc_small(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t binind)
|
|||
chunk = arena_chunk_alloc(tsdn, arena);
|
||||
if (chunk != NULL) {
|
||||
run = &arena_miscelm_get_mutable(chunk, map_bias)->run;
|
||||
if (arena_run_split_small(arena, run, size, binind))
|
||||
if (arena_run_split_small(arena, iealloc(run), run, size,
|
||||
binind))
|
||||
run = NULL;
|
||||
return (run);
|
||||
}
|
||||
|
|
@ -1435,8 +1441,9 @@ arena_dirty_count(arena_t *arena)
|
|||
npages = extent_size_get(chunkselm) >> LG_PAGE;
|
||||
chunkselm = qr_next(chunkselm, cc_link);
|
||||
} else {
|
||||
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
|
||||
rdelm);
|
||||
extent_t *extent = iealloc(rdelm);
|
||||
arena_chunk_t *chunk =
|
||||
(arena_chunk_t *)extent_addr_get(extent);
|
||||
arena_chunk_map_misc_t *miscelm =
|
||||
arena_rd_to_miscelm(rdelm);
|
||||
size_t pageind = arena_miscelm_to_pageind(miscelm);
|
||||
|
|
@ -1497,8 +1504,9 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
|
|||
LG_PAGE));
|
||||
chunkselm = chunkselm_next;
|
||||
} else {
|
||||
extent_t *extent = iealloc(rdelm);
|
||||
arena_chunk_t *chunk =
|
||||
(arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
|
||||
(arena_chunk_t *)extent_addr_get(extent);
|
||||
arena_chunk_map_misc_t *miscelm =
|
||||
arena_rd_to_miscelm(rdelm);
|
||||
size_t pageind = arena_miscelm_to_pageind(miscelm);
|
||||
|
|
@ -1523,7 +1531,8 @@ arena_stash_dirty(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
|
|||
arena_chunk_alloc(tsdn, arena);
|
||||
|
||||
/* Temporarily allocate the free dirty run. */
|
||||
arena_run_split_large(arena, run, run_size, false);
|
||||
arena_run_split_large(arena, extent, run, run_size,
|
||||
false);
|
||||
/* Stash. */
|
||||
if (false)
|
||||
qr_new(rdelm, rd_link); /* Redundant. */
|
||||
|
|
@ -1577,8 +1586,9 @@ arena_purge_stashed(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
|
|||
} else {
|
||||
size_t pageind, run_size, flag_unzeroed, flags, i;
|
||||
bool decommitted;
|
||||
extent_t *extent = iealloc(rdelm);
|
||||
arena_chunk_t *chunk =
|
||||
(arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
|
||||
(arena_chunk_t *)extent_addr_get(extent);
|
||||
arena_chunk_map_misc_t *miscelm =
|
||||
arena_rd_to_miscelm(rdelm);
|
||||
pageind = arena_miscelm_to_pageind(miscelm);
|
||||
|
|
@ -1661,8 +1671,9 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
|
|||
chunk_dalloc_wrapper(tsdn, arena, chunk_hooks, addr,
|
||||
size, zeroed, committed);
|
||||
} else {
|
||||
extent_t *extent = iealloc(rdelm);
|
||||
arena_chunk_t *chunk =
|
||||
(arena_chunk_t *)CHUNK_ADDR2BASE(rdelm);
|
||||
(arena_chunk_t *)extent_addr_get(extent);
|
||||
arena_chunk_map_misc_t *miscelm =
|
||||
arena_rd_to_miscelm(rdelm);
|
||||
size_t pageind = arena_miscelm_to_pageind(miscelm);
|
||||
|
|
@ -1670,7 +1681,7 @@ arena_unstash_purged(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks,
|
|||
pageind) != 0);
|
||||
arena_run_t *run = &miscelm->run;
|
||||
qr_remove(rdelm, rd_link);
|
||||
arena_run_dalloc(tsdn, arena, run, false, true,
|
||||
arena_run_dalloc(tsdn, arena, extent, run, false, true,
|
||||
decommitted);
|
||||
}
|
||||
}
|
||||
|
|
@ -1755,10 +1766,10 @@ arena_achunk_prof_reset(tsd_t *tsd, arena_t *arena, arena_chunk_t *chunk)
|
|||
if (arena_mapbits_large_get(chunk, pageind) != 0) {
|
||||
void *ptr = (void *)((uintptr_t)chunk + (pageind
|
||||
<< LG_PAGE));
|
||||
size_t usize = isalloc(tsd_tsdn(tsd), ptr,
|
||||
config_prof);
|
||||
size_t usize = isalloc(tsd_tsdn(tsd),
|
||||
&chunk->extent, ptr, config_prof);
|
||||
|
||||
prof_free(tsd, ptr, usize);
|
||||
prof_free(tsd, &chunk->extent, ptr, usize);
|
||||
npages = arena_mapbits_large_size_get(chunk,
|
||||
pageind) >> LG_PAGE;
|
||||
} else {
|
||||
|
|
@ -1820,12 +1831,14 @@ arena_reset(tsd_t *tsd, arena_t *arena)
|
|||
size_t usize;
|
||||
|
||||
malloc_mutex_unlock(tsd_tsdn(tsd), &arena->huge_mtx);
|
||||
if (config_stats || (config_prof && opt_prof))
|
||||
usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
|
||||
if (config_stats || (config_prof && opt_prof)) {
|
||||
usize = isalloc(tsd_tsdn(tsd), extent, ptr,
|
||||
config_prof);
|
||||
}
|
||||
/* Remove huge allocation from prof sample set. */
|
||||
if (config_prof && opt_prof)
|
||||
prof_free(tsd, ptr, usize);
|
||||
huge_dalloc(tsd_tsdn(tsd), ptr);
|
||||
prof_free(tsd, extent, ptr, usize);
|
||||
huge_dalloc(tsd_tsdn(tsd), extent, ptr);
|
||||
malloc_mutex_lock(tsd_tsdn(tsd), &arena->huge_mtx);
|
||||
/* Cancel out unwanted effects on stats. */
|
||||
if (config_stats)
|
||||
|
|
@ -1997,14 +2010,14 @@ arena_run_size_get(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
|||
}
|
||||
|
||||
static void
|
||||
arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty,
|
||||
bool cleaned, bool decommitted)
|
||||
arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent,
|
||||
arena_run_t *run, bool dirty, bool cleaned, bool decommitted)
|
||||
{
|
||||
arena_chunk_t *chunk;
|
||||
arena_chunk_map_misc_t *miscelm;
|
||||
size_t size, run_ind, run_pages, flag_dirty, flag_decommitted;
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
||||
chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
miscelm = arena_run_to_miscelm(run);
|
||||
run_ind = arena_miscelm_to_pageind(miscelm);
|
||||
assert(run_ind >= map_bias);
|
||||
|
|
@ -2074,7 +2087,7 @@ arena_run_dalloc(tsdn_t *tsdn, arena_t *arena, arena_run_t *run, bool dirty,
|
|||
|
||||
static void
|
||||
arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
arena_run_t *run, size_t oldsize, size_t newsize)
|
||||
extent_t *extent, arena_run_t *run, size_t oldsize, size_t newsize)
|
||||
{
|
||||
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
|
||||
size_t pageind = arena_miscelm_to_pageind(miscelm);
|
||||
|
|
@ -2109,13 +2122,14 @@ arena_run_trim_head(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
flag_dirty | (flag_unzeroed_mask & arena_mapbits_unzeroed_get(chunk,
|
||||
pageind+head_npages)));
|
||||
|
||||
arena_run_dalloc(tsdn, arena, run, false, false, (flag_decommitted !=
|
||||
0));
|
||||
arena_run_dalloc(tsdn, arena, extent, run, false, false,
|
||||
(flag_decommitted != 0));
|
||||
}
|
||||
|
||||
static void
|
||||
arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
arena_run_t *run, size_t oldsize, size_t newsize, bool dirty)
|
||||
extent_t *extent, arena_run_t *run, size_t oldsize, size_t newsize,
|
||||
bool dirty)
|
||||
{
|
||||
arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run);
|
||||
size_t pageind = arena_miscelm_to_pageind(miscelm);
|
||||
|
|
@ -2154,8 +2168,8 @@ arena_run_trim_tail(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
|
||||
tail_miscelm = arena_miscelm_get_mutable(chunk, pageind + head_npages);
|
||||
tail_run = &tail_miscelm->run;
|
||||
arena_run_dalloc(tsdn, arena, tail_run, dirty, false, (flag_decommitted
|
||||
!= 0));
|
||||
arena_run_dalloc(tsdn, arena, extent, tail_run, dirty, false,
|
||||
(flag_decommitted != 0));
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -2251,6 +2265,7 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin)
|
|||
assert(bin->runcur->nfree > 0);
|
||||
ret = arena_run_reg_alloc(bin->runcur, bin_info);
|
||||
if (run != NULL) {
|
||||
extent_t *extent;
|
||||
arena_chunk_t *chunk;
|
||||
|
||||
/*
|
||||
|
|
@ -2261,10 +2276,11 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin)
|
|||
* arena_bin_lower_run() must be called, as if a region
|
||||
* were just deallocated from the run.
|
||||
*/
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
||||
extent = iealloc(run);
|
||||
chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
if (run->nfree == bin_info->nregs) {
|
||||
arena_dalloc_bin_run(tsdn, arena, chunk, run,
|
||||
bin);
|
||||
arena_dalloc_bin_run(tsdn, arena, chunk, extent,
|
||||
run, bin);
|
||||
} else
|
||||
arena_bin_lower_run(arena, chunk, run, bin);
|
||||
}
|
||||
|
|
@ -2499,6 +2515,7 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||
void *ret;
|
||||
size_t alloc_size, leadsize, trailsize;
|
||||
arena_run_t *run;
|
||||
extent_t *extent;
|
||||
arena_chunk_t *chunk;
|
||||
arena_chunk_map_misc_t *miscelm;
|
||||
void *rpages;
|
||||
|
|
@ -2520,7 +2537,8 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
return (NULL);
|
||||
}
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run);
|
||||
extent = iealloc(run);
|
||||
chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
miscelm = arena_run_to_miscelm(run);
|
||||
rpages = arena_miscelm_to_rpages(miscelm);
|
||||
|
||||
|
|
@ -2531,20 +2549,22 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||
if (leadsize != 0) {
|
||||
arena_chunk_map_misc_t *head_miscelm = miscelm;
|
||||
arena_run_t *head_run = run;
|
||||
extent_t *head_extent = extent;
|
||||
|
||||
miscelm = arena_miscelm_get_mutable(chunk,
|
||||
arena_miscelm_to_pageind(head_miscelm) + (leadsize >>
|
||||
LG_PAGE));
|
||||
run = &miscelm->run;
|
||||
extent = iealloc(run);
|
||||
|
||||
arena_run_trim_head(tsdn, arena, chunk, head_run, alloc_size,
|
||||
alloc_size - leadsize);
|
||||
arena_run_trim_head(tsdn, arena, chunk, head_extent, head_run,
|
||||
alloc_size, alloc_size - leadsize);
|
||||
}
|
||||
if (trailsize != 0) {
|
||||
arena_run_trim_tail(tsdn, arena, chunk, run, usize + large_pad +
|
||||
trailsize, usize + large_pad, false);
|
||||
arena_run_trim_tail(tsdn, arena, chunk, extent, run, usize +
|
||||
large_pad + trailsize, usize + large_pad, false);
|
||||
}
|
||||
if (arena_run_init_large(arena, run, usize + large_pad, zero)) {
|
||||
if (arena_run_init_large(arena, extent, run, usize + large_pad, zero)) {
|
||||
size_t run_ind =
|
||||
arena_miscelm_to_pageind(arena_run_to_miscelm(run));
|
||||
bool dirty = (arena_mapbits_dirty_get(chunk, run_ind) != 0);
|
||||
|
|
@ -2552,7 +2572,8 @@ arena_palloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||
run_ind) != 0);
|
||||
|
||||
assert(decommitted); /* Cause of OOM. */
|
||||
arena_run_dalloc(tsdn, arena, run, dirty, false, decommitted);
|
||||
arena_run_dalloc(tsdn, arena, extent, run, dirty, false,
|
||||
decommitted);
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
return (NULL);
|
||||
}
|
||||
|
|
@ -2616,7 +2637,8 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||
}
|
||||
|
||||
void
|
||||
arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size)
|
||||
arena_prof_promoted(tsdn_t *tsdn, const extent_t *extent, const void *ptr,
|
||||
size_t size)
|
||||
{
|
||||
arena_chunk_t *chunk;
|
||||
size_t pageind;
|
||||
|
|
@ -2624,32 +2646,30 @@ arena_prof_promoted(tsdn_t *tsdn, const void *ptr, size_t size)
|
|||
|
||||
cassert(config_prof);
|
||||
assert(ptr != NULL);
|
||||
assert(CHUNK_ADDR2BASE(ptr) != ptr);
|
||||
assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS);
|
||||
assert(isalloc(tsdn, ptr, true) == LARGE_MINCLASS);
|
||||
assert(extent_addr_get(extent) != ptr);
|
||||
assert(isalloc(tsdn, extent, ptr, false) == LARGE_MINCLASS);
|
||||
assert(isalloc(tsdn, extent, ptr, true) == LARGE_MINCLASS);
|
||||
assert(size <= SMALL_MAXCLASS);
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
|
||||
binind = size2index(size);
|
||||
assert(binind < NBINS);
|
||||
arena_mapbits_large_binind_set(chunk, pageind, binind);
|
||||
|
||||
assert(isalloc(tsdn, ptr, false) == LARGE_MINCLASS);
|
||||
assert(isalloc(tsdn, ptr, true) == size);
|
||||
assert(isalloc(tsdn, extent, ptr, false) == LARGE_MINCLASS);
|
||||
assert(isalloc(tsdn, extent, ptr, true) == size);
|
||||
}
|
||||
|
||||
static void
|
||||
arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
|
||||
arena_bin_t *bin)
|
||||
arena_dissociate_bin_run(extent_t *extent, arena_run_t *run, arena_bin_t *bin)
|
||||
{
|
||||
|
||||
/* Dissociate run from bin. */
|
||||
if (run == bin->runcur)
|
||||
bin->runcur = NULL;
|
||||
else {
|
||||
szind_t binind = arena_bin_index(extent_arena_get(
|
||||
&chunk->extent), bin);
|
||||
szind_t binind = arena_bin_index(extent_arena_get(extent), bin);
|
||||
const arena_bin_info_t *bin_info = &arena_bin_info[binind];
|
||||
|
||||
/*
|
||||
|
|
@ -2668,7 +2688,7 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run,
|
|||
|
||||
static void
|
||||
arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
arena_run_t *run, arena_bin_t *bin)
|
||||
extent_t *extent, arena_run_t *run, arena_bin_t *bin)
|
||||
{
|
||||
|
||||
assert(run != bin->runcur);
|
||||
|
|
@ -2676,7 +2696,7 @@ arena_dalloc_bin_run(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
malloc_mutex_unlock(tsdn, &bin->lock);
|
||||
/******************************/
|
||||
malloc_mutex_lock(tsdn, &arena->lock);
|
||||
arena_run_dalloc(tsdn, arena, run, true, false, false);
|
||||
arena_run_dalloc(tsdn, arena, extent, run, true, false, false);
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
/****************************/
|
||||
malloc_mutex_lock(tsdn, &bin->lock);
|
||||
|
|
@ -2707,7 +2727,7 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run,
|
|||
|
||||
static void
|
||||
arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr, arena_chunk_map_bits_t *bitselm, bool junked)
|
||||
extent_t *extent, void *ptr, arena_chunk_map_bits_t *bitselm, bool junked)
|
||||
{
|
||||
size_t pageind, rpages_ind;
|
||||
arena_run_t *run;
|
||||
|
|
@ -2725,10 +2745,10 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
if (!junked && config_fill && unlikely(opt_junk_free))
|
||||
arena_dalloc_junk_small(ptr, bin_info);
|
||||
|
||||
arena_run_reg_dalloc(run, ptr);
|
||||
arena_run_reg_dalloc(run, extent, ptr);
|
||||
if (run->nfree == bin_info->nregs) {
|
||||
arena_dissociate_bin_run(chunk, run, bin);
|
||||
arena_dalloc_bin_run(tsdn, arena, chunk, run, bin);
|
||||
arena_dissociate_bin_run(extent, run, bin);
|
||||
arena_dalloc_bin_run(tsdn, arena, chunk, extent, run, bin);
|
||||
} else if (run->nfree == 1 && run != bin->runcur)
|
||||
arena_bin_lower_run(arena, chunk, run, bin);
|
||||
|
||||
|
|
@ -2740,15 +2760,17 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
|
||||
void
|
||||
arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm)
|
||||
arena_chunk_t *chunk, extent_t *extent, void *ptr,
|
||||
arena_chunk_map_bits_t *bitselm)
|
||||
{
|
||||
|
||||
arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, true);
|
||||
arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm,
|
||||
true);
|
||||
}
|
||||
|
||||
void
|
||||
arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
||||
size_t pageind, arena_chunk_map_bits_t *bitselm)
|
||||
static void
|
||||
arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
extent_t *extent, void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm)
|
||||
{
|
||||
arena_run_t *run;
|
||||
arena_bin_t *bin;
|
||||
|
|
@ -2758,13 +2780,14 @@ arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, void *ptr,
|
|||
run = &arena_miscelm_get_mutable(chunk, rpages_ind)->run;
|
||||
bin = &arena->bins[run->binind];
|
||||
malloc_mutex_lock(tsdn, &bin->lock);
|
||||
arena_dalloc_bin_locked_impl(tsdn, arena, chunk, ptr, bitselm, false);
|
||||
arena_dalloc_bin_locked_impl(tsdn, arena, chunk, extent, ptr, bitselm,
|
||||
false);
|
||||
malloc_mutex_unlock(tsdn, &bin->lock);
|
||||
}
|
||||
|
||||
void
|
||||
arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr, size_t pageind)
|
||||
extent_t *extent, void *ptr, size_t pageind)
|
||||
{
|
||||
arena_chunk_map_bits_t *bitselm;
|
||||
|
||||
|
|
@ -2774,7 +2797,7 @@ arena_dalloc_small(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
pageind)) != BININD_INVALID);
|
||||
}
|
||||
bitselm = arena_bitselm_get_mutable(chunk, pageind);
|
||||
arena_dalloc_bin(tsdn, arena, chunk, ptr, pageind, bitselm);
|
||||
arena_dalloc_bin(tsdn, arena, chunk, extent, ptr, pageind, bitselm);
|
||||
arena_decay_tick(tsdn, arena);
|
||||
}
|
||||
|
||||
|
|
@ -2798,7 +2821,7 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large =
|
|||
|
||||
static void
|
||||
arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_chunk_t *chunk, void *ptr, bool junked)
|
||||
arena_chunk_t *chunk, extent_t *extent, void *ptr, bool junked)
|
||||
{
|
||||
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
|
||||
arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk,
|
||||
|
|
@ -2821,31 +2844,31 @@ arena_dalloc_large_locked_impl(tsdn_t *tsdn, arena_t *arena,
|
|||
}
|
||||
}
|
||||
|
||||
arena_run_dalloc(tsdn, arena, run, true, false, false);
|
||||
arena_run_dalloc(tsdn, arena, extent, run, true, false, false);
|
||||
}
|
||||
|
||||
void
|
||||
arena_dalloc_large_junked_locked(tsdn_t *tsdn, arena_t *arena,
|
||||
arena_chunk_t *chunk, void *ptr)
|
||||
arena_chunk_t *chunk, extent_t *extent, void *ptr)
|
||||
{
|
||||
|
||||
arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, true);
|
||||
arena_dalloc_large_locked_impl(tsdn, arena, chunk, extent, ptr, true);
|
||||
}
|
||||
|
||||
void
|
||||
arena_dalloc_large(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr)
|
||||
extent_t *extent, void *ptr)
|
||||
{
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->lock);
|
||||
arena_dalloc_large_locked_impl(tsdn, arena, chunk, ptr, false);
|
||||
arena_dalloc_large_locked_impl(tsdn, arena, chunk, extent, ptr, false);
|
||||
malloc_mutex_unlock(tsdn, &arena->lock);
|
||||
arena_decay_tick(tsdn, arena);
|
||||
}
|
||||
|
||||
static void
|
||||
arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
||||
void *ptr, size_t oldsize, size_t size)
|
||||
extent_t *extent, void *ptr, size_t oldsize, size_t size)
|
||||
{
|
||||
size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE;
|
||||
arena_chunk_map_misc_t *miscelm = arena_miscelm_get_mutable(chunk,
|
||||
|
|
@ -2859,8 +2882,8 @@ arena_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
* allocations.
|
||||
*/
|
||||
malloc_mutex_lock(tsdn, &arena->lock);
|
||||
arena_run_trim_tail(tsdn, arena, chunk, run, oldsize + large_pad, size +
|
||||
large_pad, true);
|
||||
arena_run_trim_tail(tsdn, arena, chunk, extent, run, oldsize +
|
||||
large_pad, size + large_pad, true);
|
||||
if (config_stats) {
|
||||
szind_t oldindex = size2index(oldsize) - NBINS;
|
||||
szind_t index = size2index(size) - NBINS;
|
||||
|
|
@ -2916,7 +2939,8 @@ arena_ralloc_large_grow(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk,
|
|||
goto label_fail;
|
||||
|
||||
run = &arena_miscelm_get_mutable(chunk, pageind+npages)->run;
|
||||
if (arena_run_split_large(arena, run, splitsize, zero))
|
||||
if (arena_run_split_large(arena, iealloc(run), run, splitsize,
|
||||
zero))
|
||||
goto label_fail;
|
||||
|
||||
if (config_cache_oblivious && zero) {
|
||||
|
|
@ -3005,8 +3029,8 @@ arena_ralloc_junk_large_t *arena_ralloc_junk_large =
|
|||
* always fail if growing an object, and the following run is already in use.
|
||||
*/
|
||||
static bool
|
||||
arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
||||
size_t usize_max, bool zero)
|
||||
arena_ralloc_large(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
|
||||
size_t usize_min, size_t usize_max, bool zero)
|
||||
{
|
||||
arena_chunk_t *chunk;
|
||||
arena_t *arena;
|
||||
|
|
@ -3016,8 +3040,8 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
|||
return (false);
|
||||
}
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
arena = extent_arena_get(&chunk->extent);
|
||||
chunk = (arena_chunk_t *)extent_addr_get(extent);
|
||||
arena = extent_arena_get(extent);
|
||||
|
||||
if (oldsize < usize_max) {
|
||||
bool ret = arena_ralloc_large_grow(tsdn, arena, chunk, ptr,
|
||||
|
|
@ -3026,10 +3050,12 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
|||
if (unlikely(opt_junk_alloc)) {
|
||||
memset((void *)((uintptr_t)ptr + oldsize),
|
||||
JEMALLOC_ALLOC_JUNK,
|
||||
isalloc(tsdn, ptr, config_prof) - oldsize);
|
||||
isalloc(tsdn, extent, ptr, config_prof) -
|
||||
oldsize);
|
||||
} else if (unlikely(opt_zero)) {
|
||||
memset((void *)((uintptr_t)ptr + oldsize), 0,
|
||||
isalloc(tsdn, ptr, config_prof) - oldsize);
|
||||
isalloc(tsdn, extent, ptr, config_prof) -
|
||||
oldsize);
|
||||
}
|
||||
}
|
||||
return (ret);
|
||||
|
|
@ -3038,13 +3064,14 @@ arena_ralloc_large(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
|||
assert(oldsize > usize_max);
|
||||
/* Fill before shrinking in order avoid a race. */
|
||||
arena_ralloc_junk_large(ptr, oldsize, usize_max);
|
||||
arena_ralloc_large_shrink(tsdn, arena, chunk, ptr, oldsize, usize_max);
|
||||
arena_ralloc_large_shrink(tsdn, arena, chunk, extent, ptr, oldsize,
|
||||
usize_max);
|
||||
return (false);
|
||||
}
|
||||
|
||||
bool
|
||||
arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
|
||||
size_t extra, bool zero)
|
||||
arena_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
|
||||
size_t size, size_t extra, bool zero)
|
||||
{
|
||||
size_t usize_min, usize_max;
|
||||
|
||||
|
|
@ -3057,8 +3084,6 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
|
|||
usize_min = s2u(size);
|
||||
usize_max = s2u(size + extra);
|
||||
if (likely(oldsize <= large_maxclass && usize_min <= large_maxclass)) {
|
||||
arena_chunk_t *chunk;
|
||||
|
||||
/*
|
||||
* Avoid moving the allocation if the size class can be left the
|
||||
* same.
|
||||
|
|
@ -3073,17 +3098,16 @@ arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
|
|||
} else {
|
||||
if (usize_max <= SMALL_MAXCLASS)
|
||||
return (true);
|
||||
if (arena_ralloc_large(tsdn, ptr, oldsize, usize_min,
|
||||
usize_max, zero))
|
||||
if (arena_ralloc_large(tsdn, extent, ptr, oldsize,
|
||||
usize_min, usize_max, zero))
|
||||
return (true);
|
||||
}
|
||||
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
arena_decay_tick(tsdn, extent_arena_get(&chunk->extent));
|
||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||
return (false);
|
||||
} else {
|
||||
return (huge_ralloc_no_move(tsdn, ptr, oldsize, usize_min,
|
||||
usize_max, zero));
|
||||
return (huge_ralloc_no_move(tsdn, extent, ptr, oldsize,
|
||||
usize_min, usize_max, zero));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -3102,8 +3126,8 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|||
}
|
||||
|
||||
void *
|
||||
arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
|
||||
size_t size, size_t alignment, bool zero, tcache_t *tcache)
|
||||
arena_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
|
||||
size_t oldsize, size_t size, size_t alignment, bool zero, tcache_t *tcache)
|
||||
{
|
||||
void *ret;
|
||||
size_t usize;
|
||||
|
|
@ -3116,7 +3140,8 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
|
|||
size_t copysize;
|
||||
|
||||
/* Try to avoid moving the allocation. */
|
||||
if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero))
|
||||
if (!arena_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, 0,
|
||||
zero))
|
||||
return (ptr);
|
||||
|
||||
/*
|
||||
|
|
@ -3136,10 +3161,10 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
|
|||
|
||||
copysize = (usize < oldsize) ? usize : oldsize;
|
||||
memcpy(ret, ptr, copysize);
|
||||
isdalloct(tsdn, ptr, oldsize, tcache, true);
|
||||
isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
|
||||
} else {
|
||||
ret = huge_ralloc(tsdn, arena, ptr, oldsize, usize, alignment,
|
||||
zero, tcache);
|
||||
ret = huge_ralloc(tsdn, arena, extent, ptr, oldsize, usize,
|
||||
alignment, zero, tcache);
|
||||
}
|
||||
return (ret);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -180,6 +180,15 @@ chunk_deregister(const void *chunk, const extent_t *extent)
|
|||
}
|
||||
}
|
||||
|
||||
void
|
||||
chunk_reregister(tsdn_t *tsdn, const void *chunk, const extent_t *extent)
|
||||
{
|
||||
bool err;
|
||||
|
||||
err = chunk_register(tsdn, chunk, extent);
|
||||
assert(!err);
|
||||
}
|
||||
|
||||
/*
|
||||
* Do first-best-fit chunk selection, i.e. select the lowest chunk that best
|
||||
* fits.
|
||||
|
|
|
|||
10
src/ckh.c
10
src/ckh.c
|
|
@ -283,12 +283,12 @@ ckh_grow(tsdn_t *tsdn, ckh_t *ckh)
|
|||
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
|
||||
|
||||
if (!ckh_rebuild(ckh, tab)) {
|
||||
idalloctm(tsdn, tab, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(tab), tab, NULL, true, true);
|
||||
break;
|
||||
}
|
||||
|
||||
/* Rebuilding failed, so back out partially rebuilt table. */
|
||||
idalloctm(tsdn, ckh->tab, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
|
||||
ckh->tab = tab;
|
||||
ckh->lg_curbuckets = lg_prevbuckets;
|
||||
}
|
||||
|
|
@ -330,7 +330,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
|
|||
ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS;
|
||||
|
||||
if (!ckh_rebuild(ckh, tab)) {
|
||||
idalloctm(tsdn, tab, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(tab), tab, NULL, true, true);
|
||||
#ifdef CKH_COUNT
|
||||
ckh->nshrinks++;
|
||||
#endif
|
||||
|
|
@ -338,7 +338,7 @@ ckh_shrink(tsdn_t *tsdn, ckh_t *ckh)
|
|||
}
|
||||
|
||||
/* Rebuilding failed, so back out partially rebuilt table. */
|
||||
idalloctm(tsdn, ckh->tab, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
|
||||
ckh->tab = tab;
|
||||
ckh->lg_curbuckets = lg_prevbuckets;
|
||||
#ifdef CKH_COUNT
|
||||
|
|
@ -421,7 +421,7 @@ ckh_delete(tsdn_t *tsdn, ckh_t *ckh)
|
|||
(unsigned long long)ckh->nrelocs);
|
||||
#endif
|
||||
|
||||
idalloctm(tsdn, ckh->tab, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(ckh->tab), ckh->tab, NULL, true, true);
|
||||
if (config_debug)
|
||||
memset(ckh, JEMALLOC_FREE_JUNK, sizeof(ckh_t));
|
||||
}
|
||||
|
|
|
|||
149
src/huge.c
149
src/huge.c
|
|
@ -3,42 +3,6 @@
|
|||
|
||||
/******************************************************************************/
|
||||
|
||||
static extent_t *
|
||||
huge_extent_get(const void *ptr)
|
||||
{
|
||||
extent_t *extent;
|
||||
|
||||
extent = chunk_lookup(ptr, true);
|
||||
assert(!extent_achunk_get(extent));
|
||||
|
||||
return (extent);
|
||||
}
|
||||
|
||||
static bool
|
||||
huge_extent_set(tsdn_t *tsdn, const void *ptr, extent_t *extent)
|
||||
{
|
||||
|
||||
assert(extent_addr_get(extent) == ptr);
|
||||
assert(!extent_achunk_get(extent));
|
||||
return (chunk_register(tsdn, ptr, extent));
|
||||
}
|
||||
|
||||
static void
|
||||
huge_extent_reset(tsdn_t *tsdn, const void *ptr, extent_t *extent)
|
||||
{
|
||||
bool err;
|
||||
|
||||
err = huge_extent_set(tsdn, ptr, extent);
|
||||
assert(!err);
|
||||
}
|
||||
|
||||
static void
|
||||
huge_extent_unset(const void *ptr, const extent_t *extent)
|
||||
{
|
||||
|
||||
chunk_deregister(ptr, extent);
|
||||
}
|
||||
|
||||
void *
|
||||
huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
|
||||
{
|
||||
|
|
@ -81,15 +45,15 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
|
|||
arena = arena_choose(tsdn_tsd(tsdn), arena);
|
||||
if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(tsdn,
|
||||
arena, usize, alignment, &is_zeroed)) == NULL) {
|
||||
idalloctm(tsdn, extent, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
extent_init(extent, arena, ret, usize, is_zeroed, true);
|
||||
|
||||
if (huge_extent_set(tsdn, ret, extent)) {
|
||||
if (chunk_register(tsdn, ret, extent)) {
|
||||
arena_chunk_dalloc_huge(tsdn, arena, ret, usize);
|
||||
idalloctm(tsdn, extent, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
|
|
@ -133,11 +97,10 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
|
|||
#endif
|
||||
|
||||
static void
|
||||
huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
||||
size_t usize_min, size_t usize_max, bool zero)
|
||||
huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, void *ptr,
|
||||
size_t oldsize, size_t usize_min, size_t usize_max, bool zero)
|
||||
{
|
||||
size_t usize, usize_next;
|
||||
extent_t *extent;
|
||||
arena_t *arena;
|
||||
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
|
||||
bool pre_zeroed, post_zeroed;
|
||||
|
|
@ -150,7 +113,6 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
|||
if (oldsize == usize)
|
||||
return;
|
||||
|
||||
extent = huge_extent_get(ptr);
|
||||
arena = extent_arena_get(extent);
|
||||
pre_zeroed = extent_zeroed_get(extent);
|
||||
|
||||
|
|
@ -169,15 +131,15 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
|||
} else
|
||||
post_zeroed = pre_zeroed;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
/* Update the size of the huge allocation. */
|
||||
assert(extent_size_get(extent) != usize);
|
||||
huge_extent_unset(ptr, extent);
|
||||
chunk_deregister(tsdn, ptr, extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
extent_size_set(extent, usize);
|
||||
huge_extent_reset(tsdn, ptr, extent);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
chunk_reregister(tsdn, ptr, extent);
|
||||
/* Update zeroed. */
|
||||
extent_zeroed_set(extent, post_zeroed);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
||||
arena_chunk_ralloc_huge_similar(tsdn, arena, ptr, oldsize, usize);
|
||||
|
||||
|
|
@ -196,16 +158,14 @@ huge_ralloc_no_move_similar(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
|||
}
|
||||
|
||||
static bool
|
||||
huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
||||
size_t usize)
|
||||
huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, void *ptr,
|
||||
size_t oldsize, size_t usize)
|
||||
{
|
||||
extent_t *extent;
|
||||
arena_t *arena;
|
||||
chunk_hooks_t chunk_hooks;
|
||||
size_t cdiff;
|
||||
bool pre_zeroed, post_zeroed;
|
||||
|
||||
extent = huge_extent_get(ptr);
|
||||
arena = extent_arena_get(extent);
|
||||
pre_zeroed = extent_zeroed_get(extent);
|
||||
chunk_hooks = chunk_hooks_get(tsdn, arena);
|
||||
|
|
@ -233,14 +193,14 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
|||
} else
|
||||
post_zeroed = pre_zeroed;
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
/* Update the size of the huge allocation. */
|
||||
huge_extent_unset(ptr, extent);
|
||||
chunk_deregister(ptr, extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
extent_size_set(extent, usize);
|
||||
huge_extent_reset(tsdn, ptr, extent);
|
||||
/* Update zeroed. */
|
||||
extent_zeroed_set(extent, post_zeroed);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
chunk_reregister(tsdn, ptr, extent);
|
||||
|
||||
/* Zap the excess chunks. */
|
||||
arena_chunk_ralloc_huge_shrink(tsdn, arena, ptr, oldsize, usize);
|
||||
|
|
@ -249,14 +209,12 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
|||
}
|
||||
|
||||
static bool
|
||||
huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
||||
size_t usize, bool zero)
|
||||
huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, void *ptr,
|
||||
size_t oldsize, size_t usize, bool zero)
|
||||
{
|
||||
extent_t *extent;
|
||||
arena_t *arena;
|
||||
bool is_zeroed_subchunk, is_zeroed_chunk;
|
||||
|
||||
extent = huge_extent_get(ptr);
|
||||
arena = extent_arena_get(extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
is_zeroed_subchunk = extent_zeroed_get(extent);
|
||||
|
|
@ -272,12 +230,12 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
|||
&is_zeroed_chunk))
|
||||
return (true);
|
||||
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
/* Update the size of the huge allocation. */
|
||||
huge_extent_unset(ptr, extent);
|
||||
chunk_deregister(ptr, extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
extent_size_set(extent, usize);
|
||||
huge_extent_reset(tsdn, ptr, extent);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
chunk_reregister(tsdn, ptr, extent);
|
||||
|
||||
if (zero || (config_fill && unlikely(opt_zero))) {
|
||||
if (!is_zeroed_subchunk) {
|
||||
|
|
@ -298,8 +256,8 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, void *ptr, size_t oldsize,
|
|||
}
|
||||
|
||||
bool
|
||||
huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
||||
size_t usize_max, bool zero)
|
||||
huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t oldsize,
|
||||
size_t usize_min, size_t usize_max, bool zero)
|
||||
{
|
||||
|
||||
assert(s2u(oldsize) == oldsize);
|
||||
|
|
@ -312,16 +270,16 @@ huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
|||
|
||||
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(oldsize)) {
|
||||
/* Attempt to expand the allocation in-place. */
|
||||
if (!huge_ralloc_no_move_expand(tsdn, ptr, oldsize, usize_max,
|
||||
zero)) {
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
if (!huge_ralloc_no_move_expand(tsdn, extent, ptr, oldsize,
|
||||
usize_max, zero)) {
|
||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||
return (false);
|
||||
}
|
||||
/* Try again, this time with usize_min. */
|
||||
if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
|
||||
CHUNK_CEILING(oldsize) && huge_ralloc_no_move_expand(tsdn,
|
||||
ptr, oldsize, usize_min, zero)) {
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
extent, ptr, oldsize, usize_min, zero)) {
|
||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
|
@ -332,17 +290,17 @@ huge_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t usize_min,
|
|||
*/
|
||||
if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize_min)
|
||||
&& CHUNK_CEILING(oldsize) <= CHUNK_CEILING(usize_max)) {
|
||||
huge_ralloc_no_move_similar(tsdn, ptr, oldsize, usize_min,
|
||||
usize_max, zero);
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
huge_ralloc_no_move_similar(tsdn, extent, ptr, oldsize,
|
||||
usize_min, usize_max, zero);
|
||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||
return (false);
|
||||
}
|
||||
|
||||
/* Attempt to shrink the allocation in-place. */
|
||||
if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize_max)) {
|
||||
if (!huge_ralloc_no_move_shrink(tsdn, ptr, oldsize,
|
||||
if (!huge_ralloc_no_move_shrink(tsdn, extent, ptr, oldsize,
|
||||
usize_max)) {
|
||||
arena_decay_tick(tsdn, huge_aalloc(ptr));
|
||||
arena_decay_tick(tsdn, extent_arena_get(extent));
|
||||
return (false);
|
||||
}
|
||||
}
|
||||
|
|
@ -360,8 +318,8 @@ huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
|
|||
}
|
||||
|
||||
void *
|
||||
huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
|
||||
size_t usize, size_t alignment, bool zero, tcache_t *tcache)
|
||||
huge_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr,
|
||||
size_t oldsize, size_t usize, size_t alignment, bool zero, tcache_t *tcache)
|
||||
{
|
||||
void *ret;
|
||||
size_t copysize;
|
||||
|
|
@ -370,7 +328,8 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
|
|||
assert(usize > 0 && usize <= HUGE_MAXCLASS);
|
||||
|
||||
/* Try to avoid moving the allocation. */
|
||||
if (!huge_ralloc_no_move(tsdn, ptr, oldsize, usize, usize, zero))
|
||||
if (!huge_ralloc_no_move(tsdn, extent, ptr, oldsize, usize, usize,
|
||||
zero))
|
||||
return (ptr);
|
||||
|
||||
/*
|
||||
|
|
@ -384,19 +343,17 @@ huge_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
|
|||
|
||||
copysize = (usize < oldsize) ? usize : oldsize;
|
||||
memcpy(ret, ptr, copysize);
|
||||
isdalloct(tsdn, ptr, oldsize, tcache, true);
|
||||
isdalloct(tsdn, extent, ptr, oldsize, tcache, true);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
void
|
||||
huge_dalloc(tsdn_t *tsdn, void *ptr)
|
||||
huge_dalloc(tsdn_t *tsdn, extent_t *extent, void *ptr)
|
||||
{
|
||||
extent_t *extent;
|
||||
arena_t *arena;
|
||||
|
||||
extent = huge_extent_get(ptr);
|
||||
arena = extent_arena_get(extent);
|
||||
huge_extent_unset(ptr, extent);
|
||||
chunk_deregister(ptr, extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
ql_remove(&arena->huge, extent, ql_link);
|
||||
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
|
||||
|
|
@ -405,26 +362,17 @@ huge_dalloc(tsdn_t *tsdn, void *ptr)
|
|||
extent_size_get(extent));
|
||||
arena_chunk_dalloc_huge(tsdn, extent_arena_get(extent),
|
||||
extent_addr_get(extent), extent_size_get(extent));
|
||||
idalloctm(tsdn, extent, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(extent), extent, NULL, true, true);
|
||||
|
||||
arena_decay_tick(tsdn, arena);
|
||||
}
|
||||
|
||||
arena_t *
|
||||
huge_aalloc(const void *ptr)
|
||||
{
|
||||
|
||||
return (extent_arena_get(huge_extent_get(ptr)));
|
||||
}
|
||||
|
||||
size_t
|
||||
huge_salloc(tsdn_t *tsdn, const void *ptr)
|
||||
huge_salloc(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
|
||||
{
|
||||
size_t size;
|
||||
extent_t *extent;
|
||||
arena_t *arena;
|
||||
|
||||
extent = huge_extent_get(ptr);
|
||||
arena = extent_arena_get(extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
size = extent_size_get(extent);
|
||||
|
|
@ -434,13 +382,13 @@ huge_salloc(tsdn_t *tsdn, const void *ptr)
|
|||
}
|
||||
|
||||
prof_tctx_t *
|
||||
huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
|
||||
huge_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent, const void *ptr)
|
||||
{
|
||||
prof_tctx_t *tctx;
|
||||
extent_t *extent;
|
||||
arena_t *arena;
|
||||
|
||||
extent = huge_extent_get(ptr);
|
||||
assert(extent == iealloc(ptr));
|
||||
|
||||
arena = extent_arena_get(extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
tctx = extent_prof_tctx_get(extent);
|
||||
|
|
@ -450,12 +398,13 @@ huge_prof_tctx_get(tsdn_t *tsdn, const void *ptr)
|
|||
}
|
||||
|
||||
void
|
||||
huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
|
||||
huge_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, const void *ptr,
|
||||
prof_tctx_t *tctx)
|
||||
{
|
||||
extent_t *extent;
|
||||
arena_t *arena;
|
||||
|
||||
extent = huge_extent_get(ptr);
|
||||
assert(extent == iealloc(ptr));
|
||||
|
||||
arena = extent_arena_get(extent);
|
||||
malloc_mutex_lock(tsdn, &arena->huge_mtx);
|
||||
extent_prof_tctx_set(extent, tctx);
|
||||
|
|
@ -463,8 +412,8 @@ huge_prof_tctx_set(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx)
|
|||
}
|
||||
|
||||
void
|
||||
huge_prof_tctx_reset(tsdn_t *tsdn, const void *ptr)
|
||||
huge_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent, const void *ptr)
|
||||
{
|
||||
|
||||
huge_prof_tctx_set(tsdn, ptr, (prof_tctx_t *)(uintptr_t)1U);
|
||||
huge_prof_tctx_set(tsdn, extent, ptr, (prof_tctx_t *)(uintptr_t)1U);
|
||||
}
|
||||
|
|
|
|||
210
src/jemalloc.c
210
src/jemalloc.c
|
|
@ -308,10 +308,10 @@ a0ialloc(size_t size, bool zero, bool is_metadata)
|
|||
}
|
||||
|
||||
static void
|
||||
a0idalloc(void *ptr, bool is_metadata)
|
||||
a0idalloc(extent_t *extent, void *ptr, bool is_metadata)
|
||||
{
|
||||
|
||||
idalloctm(TSDN_NULL, ptr, false, is_metadata, true);
|
||||
idalloctm(TSDN_NULL, extent, ptr, false, is_metadata, true);
|
||||
}
|
||||
|
||||
void *
|
||||
|
|
@ -325,7 +325,7 @@ void
|
|||
a0dalloc(void *ptr)
|
||||
{
|
||||
|
||||
a0idalloc(ptr, true);
|
||||
a0idalloc(iealloc(ptr), ptr, true);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -365,7 +365,7 @@ bootstrap_free(void *ptr)
|
|||
if (unlikely(ptr == NULL))
|
||||
return;
|
||||
|
||||
a0idalloc(ptr, false);
|
||||
a0idalloc(iealloc(ptr), ptr, false);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -1401,7 +1401,7 @@ ialloc_prof_sample(tsd_t *tsd, size_t usize, szind_t ind, bool zero,
|
|||
p = ialloc(tsd, LARGE_MINCLASS, ind_large, zero, slow_path);
|
||||
if (p == NULL)
|
||||
return (NULL);
|
||||
arena_prof_promoted(tsd_tsdn(tsd), p, usize);
|
||||
arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
|
||||
} else
|
||||
p = ialloc(tsd, usize, ind, zero, slow_path);
|
||||
|
||||
|
|
@ -1423,7 +1423,7 @@ ialloc_prof(tsd_t *tsd, size_t usize, szind_t ind, bool zero, bool slow_path)
|
|||
prof_alloc_rollback(tsd, tctx, true);
|
||||
return (NULL);
|
||||
}
|
||||
prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
|
||||
prof_malloc(tsd_tsdn(tsd), iealloc(p), p, usize, tctx);
|
||||
|
||||
return (p);
|
||||
}
|
||||
|
|
@ -1482,7 +1482,7 @@ ialloc_post_check(void *ret, tsdn_t *tsdn, size_t usize, const char *func,
|
|||
set_errno(ENOMEM);
|
||||
}
|
||||
if (config_stats && likely(ret != NULL)) {
|
||||
assert(usize == isalloc(tsdn, ret, config_prof));
|
||||
assert(usize == isalloc(tsdn, iealloc(ret), ret, config_prof));
|
||||
*tsd_thread_allocatedp_get(tsdn_tsd(tsdn)) += usize;
|
||||
}
|
||||
witness_assert_lockless(tsdn);
|
||||
|
|
@ -1525,7 +1525,7 @@ imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize,
|
|||
p = ipalloc(tsd, LARGE_MINCLASS, alignment, false);
|
||||
if (p == NULL)
|
||||
return (NULL);
|
||||
arena_prof_promoted(tsd_tsdn(tsd), p, usize);
|
||||
arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
|
||||
} else
|
||||
p = ipalloc(tsd, usize, alignment, false);
|
||||
|
||||
|
|
@ -1547,7 +1547,7 @@ imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize)
|
|||
prof_alloc_rollback(tsd, tctx, true);
|
||||
return (NULL);
|
||||
}
|
||||
prof_malloc(tsd_tsdn(tsd), p, usize, tctx);
|
||||
prof_malloc(tsd_tsdn(tsd), iealloc(p), p, usize, tctx);
|
||||
|
||||
return (p);
|
||||
}
|
||||
|
|
@ -1604,7 +1604,8 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment)
|
|||
ret = 0;
|
||||
label_return:
|
||||
if (config_stats && likely(result != NULL)) {
|
||||
assert(usize == isalloc(tsd_tsdn(tsd), result, config_prof));
|
||||
assert(usize == isalloc(tsd_tsdn(tsd), iealloc(result), result,
|
||||
config_prof));
|
||||
*tsd_thread_allocatedp_get(tsd) += usize;
|
||||
}
|
||||
UTRACE(0, size, result);
|
||||
|
|
@ -1683,44 +1684,49 @@ je_calloc(size_t num, size_t size)
|
|||
}
|
||||
|
||||
static void *
|
||||
irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize,
|
||||
prof_tctx_t *tctx)
|
||||
irealloc_prof_sample(tsd_t *tsd, extent_t *extent, void *old_ptr,
|
||||
size_t old_usize, size_t usize, prof_tctx_t *tctx)
|
||||
{
|
||||
void *p;
|
||||
|
||||
if (tctx == NULL)
|
||||
return (NULL);
|
||||
if (usize <= SMALL_MAXCLASS) {
|
||||
p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false);
|
||||
p = iralloc(tsd, extent, old_ptr, old_usize, LARGE_MINCLASS, 0,
|
||||
false);
|
||||
if (p == NULL)
|
||||
return (NULL);
|
||||
arena_prof_promoted(tsd_tsdn(tsd), p, usize);
|
||||
arena_prof_promoted(tsd_tsdn(tsd), iealloc(p), p, usize);
|
||||
} else
|
||||
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
|
||||
p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false);
|
||||
|
||||
return (p);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE_C void *
|
||||
irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
|
||||
irealloc_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize,
|
||||
size_t usize)
|
||||
{
|
||||
void *p;
|
||||
extent_t *e;
|
||||
bool prof_active;
|
||||
prof_tctx_t *old_tctx, *tctx;
|
||||
|
||||
prof_active = prof_active_get_unlocked();
|
||||
old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
|
||||
old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr);
|
||||
tctx = prof_alloc_prep(tsd, usize, prof_active, true);
|
||||
if (unlikely((uintptr_t)tctx != (uintptr_t)1U))
|
||||
p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx);
|
||||
else
|
||||
p = iralloc(tsd, old_ptr, old_usize, usize, 0, false);
|
||||
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
|
||||
p = irealloc_prof_sample(tsd, extent, old_ptr, old_usize, usize,
|
||||
tctx);
|
||||
} else
|
||||
p = iralloc(tsd, extent, old_ptr, old_usize, usize, 0, false);
|
||||
if (unlikely(p == NULL)) {
|
||||
prof_alloc_rollback(tsd, tctx, true);
|
||||
return (NULL);
|
||||
}
|
||||
prof_realloc(tsd, p, usize, tctx, prof_active, true, old_ptr, old_usize,
|
||||
old_tctx);
|
||||
e = (p == old_ptr) ? extent : iealloc(p);
|
||||
prof_realloc(tsd, e, p, usize, tctx, prof_active, true,
|
||||
old_ptr, old_usize, old_tctx);
|
||||
|
||||
return (p);
|
||||
}
|
||||
|
|
@ -1728,6 +1734,7 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize)
|
|||
JEMALLOC_INLINE_C void
|
||||
ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
|
||||
{
|
||||
extent_t *extent;
|
||||
size_t usize;
|
||||
|
||||
witness_assert_lockless(tsd_tsdn(tsd));
|
||||
|
|
@ -1735,22 +1742,24 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path)
|
|||
assert(ptr != NULL);
|
||||
assert(malloc_initialized() || IS_INITIALIZER);
|
||||
|
||||
extent = iealloc(ptr);
|
||||
if (config_prof && opt_prof) {
|
||||
usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
|
||||
prof_free(tsd, ptr, usize);
|
||||
usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
|
||||
prof_free(tsd, extent, ptr, usize);
|
||||
} else if (config_stats)
|
||||
usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
|
||||
usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
|
||||
if (config_stats)
|
||||
*tsd_thread_deallocatedp_get(tsd) += usize;
|
||||
|
||||
if (likely(!slow_path))
|
||||
idalloctm(tsd_tsdn(tsd), ptr, tcache, false, false);
|
||||
idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, false);
|
||||
else
|
||||
idalloctm(tsd_tsdn(tsd), ptr, tcache, false, true);
|
||||
idalloctm(tsd_tsdn(tsd), extent, ptr, tcache, false, true);
|
||||
}
|
||||
|
||||
JEMALLOC_INLINE_C void
|
||||
isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
|
||||
isfree(tsd_t *tsd, extent_t *extent, void *ptr, size_t usize, tcache_t *tcache,
|
||||
bool slow_path)
|
||||
{
|
||||
|
||||
witness_assert_lockless(tsd_tsdn(tsd));
|
||||
|
|
@ -1759,14 +1768,14 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path)
|
|||
assert(malloc_initialized() || IS_INITIALIZER);
|
||||
|
||||
if (config_prof && opt_prof)
|
||||
prof_free(tsd, ptr, usize);
|
||||
prof_free(tsd, extent, ptr, usize);
|
||||
if (config_stats)
|
||||
*tsd_thread_deallocatedp_get(tsd) += usize;
|
||||
|
||||
if (likely(!slow_path))
|
||||
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, false);
|
||||
isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, false);
|
||||
else
|
||||
isdalloct(tsd_tsdn(tsd), ptr, usize, tcache, true);
|
||||
isdalloct(tsd_tsdn(tsd), extent, ptr, usize, tcache, true);
|
||||
}
|
||||
|
||||
JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN
|
||||
|
|
@ -1794,22 +1803,26 @@ je_realloc(void *ptr, size_t size)
|
|||
|
||||
if (likely(ptr != NULL)) {
|
||||
tsd_t *tsd;
|
||||
extent_t *extent;
|
||||
|
||||
assert(malloc_initialized() || IS_INITIALIZER);
|
||||
tsd = tsd_fetch();
|
||||
|
||||
witness_assert_lockless(tsd_tsdn(tsd));
|
||||
|
||||
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
|
||||
extent = iealloc(ptr);
|
||||
old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
|
||||
|
||||
if (config_prof && opt_prof) {
|
||||
usize = s2u(size);
|
||||
ret = unlikely(usize == 0 || usize > HUGE_MAXCLASS) ?
|
||||
NULL : irealloc_prof(tsd, ptr, old_usize, usize);
|
||||
NULL : irealloc_prof(tsd, extent, ptr, old_usize,
|
||||
usize);
|
||||
} else {
|
||||
if (config_stats)
|
||||
usize = s2u(size);
|
||||
ret = iralloc(tsd, ptr, old_usize, size, 0, false);
|
||||
ret = iralloc(tsd, extent, ptr, old_usize, size, 0,
|
||||
false);
|
||||
}
|
||||
tsdn = tsd_tsdn(tsd);
|
||||
} else {
|
||||
|
|
@ -1832,7 +1845,7 @@ je_realloc(void *ptr, size_t size)
|
|||
if (config_stats && likely(ret != NULL)) {
|
||||
tsd_t *tsd;
|
||||
|
||||
assert(usize == isalloc(tsdn, ret, config_prof));
|
||||
assert(usize == isalloc(tsdn, iealloc(ret), ret, config_prof));
|
||||
tsd = tsdn_tsd(tsdn);
|
||||
*tsd_thread_allocatedp_get(tsd) += usize;
|
||||
*tsd_thread_deallocatedp_get(tsd) += old_usize;
|
||||
|
|
@ -1986,11 +1999,10 @@ imallocx_prof_sample(tsdn_t *tsdn, size_t usize, size_t alignment, bool zero,
|
|||
tcache, arena, slow_path);
|
||||
if (p == NULL)
|
||||
return (NULL);
|
||||
arena_prof_promoted(tsdn, p, usize);
|
||||
} else {
|
||||
arena_prof_promoted(tsdn, iealloc(p), p, usize);
|
||||
} else
|
||||
p = imallocx_flags(tsdn, usize, alignment, zero, tcache, arena,
|
||||
slow_path);
|
||||
}
|
||||
|
||||
return (p);
|
||||
}
|
||||
|
|
@ -2021,7 +2033,7 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize, bool slow_path)
|
|||
prof_alloc_rollback(tsd, tctx, true);
|
||||
return (NULL);
|
||||
}
|
||||
prof_malloc(tsd_tsdn(tsd), p, *usize, tctx);
|
||||
prof_malloc(tsd_tsdn(tsd), iealloc(p), p, *usize, tctx);
|
||||
|
||||
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
|
||||
return (p);
|
||||
|
|
@ -2109,46 +2121,47 @@ je_mallocx(size_t size, int flags)
|
|||
}
|
||||
|
||||
static void *
|
||||
irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize,
|
||||
size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena,
|
||||
prof_tctx_t *tctx)
|
||||
irallocx_prof_sample(tsdn_t *tsdn, extent_t *extent, void *old_ptr,
|
||||
size_t old_usize, size_t usize, size_t alignment, bool zero,
|
||||
tcache_t *tcache, arena_t *arena, prof_tctx_t *tctx)
|
||||
{
|
||||
void *p;
|
||||
|
||||
if (tctx == NULL)
|
||||
return (NULL);
|
||||
if (usize <= SMALL_MAXCLASS) {
|
||||
p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS,
|
||||
p = iralloct(tsdn, extent, old_ptr, old_usize, LARGE_MINCLASS,
|
||||
alignment, zero, tcache, arena);
|
||||
if (p == NULL)
|
||||
return (NULL);
|
||||
arena_prof_promoted(tsdn, p, usize);
|
||||
arena_prof_promoted(tsdn, iealloc(p), p, usize);
|
||||
} else {
|
||||
p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero,
|
||||
tcache, arena);
|
||||
p = iralloct(tsdn, extent, old_ptr, old_usize, usize, alignment,
|
||||
zero, tcache, arena);
|
||||
}
|
||||
|
||||
return (p);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE_C void *
|
||||
irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
|
||||
size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
|
||||
irallocx_prof(tsd_t *tsd, extent_t *extent, void *old_ptr, size_t old_usize,
|
||||
size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache,
|
||||
arena_t *arena)
|
||||
{
|
||||
void *p;
|
||||
extent_t *e;
|
||||
bool prof_active;
|
||||
prof_tctx_t *old_tctx, *tctx;
|
||||
|
||||
prof_active = prof_active_get_unlocked();
|
||||
old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr);
|
||||
old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, old_ptr);
|
||||
tctx = prof_alloc_prep(tsd, *usize, prof_active, true);
|
||||
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
|
||||
p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize,
|
||||
*usize, alignment, zero, tcache, arena, tctx);
|
||||
p = irallocx_prof_sample(tsd_tsdn(tsd), extent, old_ptr,
|
||||
old_usize, *usize, alignment, zero, tcache, arena, tctx);
|
||||
} else {
|
||||
p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment,
|
||||
zero, tcache, arena);
|
||||
p = iralloct(tsd_tsdn(tsd), extent, old_ptr, old_usize, size,
|
||||
alignment, zero, tcache, arena);
|
||||
}
|
||||
if (unlikely(p == NULL)) {
|
||||
prof_alloc_rollback(tsd, tctx, true);
|
||||
|
|
@ -2164,9 +2177,11 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size,
|
|||
* be the same as the current usize because of in-place large
|
||||
* reallocation. Therefore, query the actual value of usize.
|
||||
*/
|
||||
*usize = isalloc(tsd_tsdn(tsd), p, config_prof);
|
||||
}
|
||||
prof_realloc(tsd, p, *usize, tctx, prof_active, true, old_ptr,
|
||||
e = extent;
|
||||
*usize = isalloc(tsd_tsdn(tsd), e, p, config_prof);
|
||||
} else
|
||||
e = iealloc(p);
|
||||
prof_realloc(tsd, e, p, *usize, tctx, prof_active, true, old_ptr,
|
||||
old_usize, old_tctx);
|
||||
|
||||
return (p);
|
||||
|
|
@ -2179,6 +2194,7 @@ je_rallocx(void *ptr, size_t size, int flags)
|
|||
{
|
||||
void *p;
|
||||
tsd_t *tsd;
|
||||
extent_t *extent;
|
||||
size_t usize;
|
||||
size_t old_usize;
|
||||
size_t alignment = MALLOCX_ALIGN_GET(flags);
|
||||
|
|
@ -2191,6 +2207,7 @@ je_rallocx(void *ptr, size_t size, int flags)
|
|||
assert(malloc_initialized() || IS_INITIALIZER);
|
||||
tsd = tsd_fetch();
|
||||
witness_assert_lockless(tsd_tsdn(tsd));
|
||||
extent = iealloc(ptr);
|
||||
|
||||
if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) {
|
||||
unsigned arena_ind = MALLOCX_ARENA_GET(flags);
|
||||
|
|
@ -2208,23 +2225,25 @@ je_rallocx(void *ptr, size_t size, int flags)
|
|||
} else
|
||||
tcache = tcache_get(tsd, true);
|
||||
|
||||
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
|
||||
old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
|
||||
|
||||
if (config_prof && opt_prof) {
|
||||
usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment);
|
||||
if (unlikely(usize == 0 || usize > HUGE_MAXCLASS))
|
||||
goto label_oom;
|
||||
p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize,
|
||||
zero, tcache, arena);
|
||||
p = irallocx_prof(tsd, extent, ptr, old_usize, size, alignment,
|
||||
&usize, zero, tcache, arena);
|
||||
if (unlikely(p == NULL))
|
||||
goto label_oom;
|
||||
} else {
|
||||
p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment,
|
||||
zero, tcache, arena);
|
||||
p = iralloct(tsd_tsdn(tsd), extent, ptr, old_usize, size,
|
||||
alignment, zero, tcache, arena);
|
||||
if (unlikely(p == NULL))
|
||||
goto label_oom;
|
||||
if (config_stats)
|
||||
usize = isalloc(tsd_tsdn(tsd), p, config_prof);
|
||||
if (config_stats) {
|
||||
usize = isalloc(tsd_tsdn(tsd), iealloc(p), p,
|
||||
config_prof);
|
||||
}
|
||||
}
|
||||
assert(alignment == 0 || ((uintptr_t)p & (alignment - 1)) == ZU(0));
|
||||
|
||||
|
|
@ -2246,42 +2265,43 @@ label_oom:
|
|||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE_C size_t
|
||||
ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
|
||||
size_t extra, size_t alignment, bool zero)
|
||||
ixallocx_helper(tsdn_t *tsdn, extent_t *extent, void *ptr, size_t old_usize,
|
||||
size_t size, size_t extra, size_t alignment, bool zero)
|
||||
{
|
||||
size_t usize;
|
||||
|
||||
if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero))
|
||||
if (ixalloc(tsdn, extent, ptr, old_usize, size, extra, alignment, zero))
|
||||
return (old_usize);
|
||||
usize = isalloc(tsdn, ptr, config_prof);
|
||||
usize = isalloc(tsdn, extent, ptr, config_prof);
|
||||
|
||||
return (usize);
|
||||
}
|
||||
|
||||
static size_t
|
||||
ixallocx_prof_sample(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size,
|
||||
size_t extra, size_t alignment, bool zero, prof_tctx_t *tctx)
|
||||
ixallocx_prof_sample(tsdn_t *tsdn, extent_t *extent, void *ptr,
|
||||
size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero,
|
||||
prof_tctx_t *tctx)
|
||||
{
|
||||
size_t usize;
|
||||
|
||||
if (tctx == NULL)
|
||||
return (old_usize);
|
||||
usize = ixallocx_helper(tsdn, ptr, old_usize, size, extra, alignment,
|
||||
zero);
|
||||
usize = ixallocx_helper(tsdn, extent, ptr, old_usize, size, extra,
|
||||
alignment, zero);
|
||||
|
||||
return (usize);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE_C size_t
|
||||
ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
|
||||
size_t extra, size_t alignment, bool zero)
|
||||
ixallocx_prof(tsd_t *tsd, extent_t *extent, void *ptr, size_t old_usize,
|
||||
size_t size, size_t extra, size_t alignment, bool zero)
|
||||
{
|
||||
size_t usize_max, usize;
|
||||
bool prof_active;
|
||||
prof_tctx_t *old_tctx, *tctx;
|
||||
|
||||
prof_active = prof_active_get_unlocked();
|
||||
old_tctx = prof_tctx_get(tsd_tsdn(tsd), ptr);
|
||||
old_tctx = prof_tctx_get(tsd_tsdn(tsd), extent, ptr);
|
||||
/*
|
||||
* usize isn't knowable before ixalloc() returns when extra is non-zero.
|
||||
* Therefore, compute its maximum possible value and use that in
|
||||
|
|
@ -2306,18 +2326,18 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size,
|
|||
tctx = prof_alloc_prep(tsd, usize_max, prof_active, false);
|
||||
|
||||
if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) {
|
||||
usize = ixallocx_prof_sample(tsd_tsdn(tsd), ptr, old_usize,
|
||||
size, extra, alignment, zero, tctx);
|
||||
usize = ixallocx_prof_sample(tsd_tsdn(tsd), extent, ptr,
|
||||
old_usize, size, extra, alignment, zero, tctx);
|
||||
} else {
|
||||
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
|
||||
extra, alignment, zero);
|
||||
usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize,
|
||||
size, extra, alignment, zero);
|
||||
}
|
||||
if (usize == old_usize) {
|
||||
prof_alloc_rollback(tsd, tctx, false);
|
||||
return (usize);
|
||||
}
|
||||
prof_realloc(tsd, ptr, usize, tctx, prof_active, false, ptr, old_usize,
|
||||
old_tctx);
|
||||
prof_realloc(tsd, extent, ptr, usize, tctx, prof_active, false, ptr,
|
||||
old_usize, old_tctx);
|
||||
|
||||
return (usize);
|
||||
}
|
||||
|
|
@ -2326,6 +2346,7 @@ JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW
|
|||
je_xallocx(void *ptr, size_t size, size_t extra, int flags)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
extent_t *extent;
|
||||
size_t usize, old_usize;
|
||||
size_t alignment = MALLOCX_ALIGN_GET(flags);
|
||||
bool zero = flags & MALLOCX_ZERO;
|
||||
|
|
@ -2336,8 +2357,9 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
|
|||
assert(malloc_initialized() || IS_INITIALIZER);
|
||||
tsd = tsd_fetch();
|
||||
witness_assert_lockless(tsd_tsdn(tsd));
|
||||
extent = iealloc(ptr);
|
||||
|
||||
old_usize = isalloc(tsd_tsdn(tsd), ptr, config_prof);
|
||||
old_usize = isalloc(tsd_tsdn(tsd), extent, ptr, config_prof);
|
||||
|
||||
/*
|
||||
* The API explicitly absolves itself of protecting against (size +
|
||||
|
|
@ -2356,11 +2378,11 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags)
|
|||
extra = HUGE_MAXCLASS - size;
|
||||
|
||||
if (config_prof && opt_prof) {
|
||||
usize = ixallocx_prof(tsd, ptr, old_usize, size, extra,
|
||||
usize = ixallocx_prof(tsd, extent, ptr, old_usize, size, extra,
|
||||
alignment, zero);
|
||||
} else {
|
||||
usize = ixallocx_helper(tsd_tsdn(tsd), ptr, old_usize, size,
|
||||
extra, alignment, zero);
|
||||
usize = ixallocx_helper(tsd_tsdn(tsd), extent, ptr, old_usize,
|
||||
size, extra, alignment, zero);
|
||||
}
|
||||
if (unlikely(usize == old_usize))
|
||||
goto label_not_resized;
|
||||
|
|
@ -2390,7 +2412,7 @@ je_sallocx(const void *ptr, int flags)
|
|||
if (config_ivsalloc)
|
||||
usize = ivsalloc(tsdn, ptr, config_prof);
|
||||
else
|
||||
usize = isalloc(tsdn, ptr, config_prof);
|
||||
usize = isalloc(tsdn, iealloc(ptr), ptr, config_prof);
|
||||
|
||||
witness_assert_lockless(tsdn);
|
||||
return (usize);
|
||||
|
|
@ -2442,14 +2464,16 @@ JEMALLOC_EXPORT void JEMALLOC_NOTHROW
|
|||
je_sdallocx(void *ptr, size_t size, int flags)
|
||||
{
|
||||
tsd_t *tsd;
|
||||
tcache_t *tcache;
|
||||
extent_t *extent;
|
||||
size_t usize;
|
||||
tcache_t *tcache;
|
||||
|
||||
assert(ptr != NULL);
|
||||
assert(malloc_initialized() || IS_INITIALIZER);
|
||||
tsd = tsd_fetch();
|
||||
extent = iealloc(ptr);
|
||||
usize = inallocx(tsd_tsdn(tsd), size, flags);
|
||||
assert(usize == isalloc(tsd_tsdn(tsd), ptr, config_prof));
|
||||
assert(usize == isalloc(tsd_tsdn(tsd), extent, ptr, config_prof));
|
||||
|
||||
witness_assert_lockless(tsd_tsdn(tsd));
|
||||
if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) {
|
||||
|
|
@ -2462,9 +2486,9 @@ je_sdallocx(void *ptr, size_t size, int flags)
|
|||
|
||||
UTRACE(ptr, 0, 0);
|
||||
if (likely(!malloc_slow))
|
||||
isfree(tsd, ptr, usize, tcache, false);
|
||||
isfree(tsd, extent, ptr, usize, tcache, false);
|
||||
else
|
||||
isfree(tsd, ptr, usize, tcache, true);
|
||||
isfree(tsd, extent, ptr, usize, tcache, true);
|
||||
witness_assert_lockless(tsd_tsdn(tsd));
|
||||
}
|
||||
|
||||
|
|
@ -2566,8 +2590,10 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr)
|
|||
|
||||
if (config_ivsalloc)
|
||||
ret = ivsalloc(tsdn, ptr, config_prof);
|
||||
else
|
||||
ret = (ptr == NULL) ? 0 : isalloc(tsdn, ptr, config_prof);
|
||||
else {
|
||||
ret = (ptr == NULL) ? 0 : isalloc(tsdn, iealloc(ptr), ptr,
|
||||
config_prof);
|
||||
}
|
||||
|
||||
witness_assert_lockless(tsdn);
|
||||
return (ret);
|
||||
|
|
|
|||
32
src/prof.c
32
src/prof.c
|
|
@ -223,11 +223,11 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated)
|
|||
}
|
||||
|
||||
void
|
||||
prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize,
|
||||
prof_tctx_t *tctx)
|
||||
prof_malloc_sample_object(tsdn_t *tsdn, extent_t *extent, const void *ptr,
|
||||
size_t usize, prof_tctx_t *tctx)
|
||||
{
|
||||
|
||||
prof_tctx_set(tsdn, ptr, usize, tctx);
|
||||
prof_tctx_set(tsdn, extent, ptr, usize, tctx);
|
||||
|
||||
malloc_mutex_lock(tsdn, tctx->tdata->lock);
|
||||
tctx->cnts.curobjs++;
|
||||
|
|
@ -596,7 +596,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx,
|
|||
prof_leave(tsd, tdata_self);
|
||||
/* Destroy gctx. */
|
||||
malloc_mutex_unlock(tsd_tsdn(tsd), gctx->lock);
|
||||
idalloctm(tsd_tsdn(tsd), gctx, NULL, true, true);
|
||||
idalloctm(tsd_tsdn(tsd), iealloc(gctx), gctx, NULL, true, true);
|
||||
} else {
|
||||
/*
|
||||
* Compensate for increment in prof_tctx_destroy() or
|
||||
|
|
@ -707,7 +707,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx)
|
|||
prof_tdata_destroy(tsd_tsdn(tsd), tdata, false);
|
||||
|
||||
if (destroy_tctx)
|
||||
idalloctm(tsd_tsdn(tsd), tctx, NULL, true, true);
|
||||
idalloctm(tsd_tsdn(tsd), iealloc(tctx), tctx, NULL, true, true);
|
||||
}
|
||||
|
||||
static bool
|
||||
|
|
@ -736,7 +736,8 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata,
|
|||
if (ckh_insert(tsd_tsdn(tsd), &bt2gctx, btkey.v, gctx.v)) {
|
||||
/* OOM. */
|
||||
prof_leave(tsd, tdata);
|
||||
idalloctm(tsd_tsdn(tsd), gctx.v, NULL, true, true);
|
||||
idalloctm(tsd_tsdn(tsd), iealloc(gctx.v), gctx.v, NULL,
|
||||
true, true);
|
||||
return (true);
|
||||
}
|
||||
new_gctx = true;
|
||||
|
|
@ -816,7 +817,8 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt)
|
|||
if (error) {
|
||||
if (new_gctx)
|
||||
prof_gctx_try_destroy(tsd, tdata, gctx, tdata);
|
||||
idalloctm(tsd_tsdn(tsd), ret.v, NULL, true, true);
|
||||
idalloctm(tsd_tsdn(tsd), iealloc(ret.v), ret.v, NULL,
|
||||
true, true);
|
||||
return (NULL);
|
||||
}
|
||||
malloc_mutex_lock(tsd_tsdn(tsd), gctx->lock);
|
||||
|
|
@ -1238,7 +1240,8 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs)
|
|||
to_destroy);
|
||||
tctx_tree_remove(&gctx->tctxs,
|
||||
to_destroy);
|
||||
idalloctm(tsd_tsdn(tsd), to_destroy,
|
||||
idalloctm(tsd_tsdn(tsd),
|
||||
iealloc(to_destroy), to_destroy,
|
||||
NULL, true, true);
|
||||
} else
|
||||
next = NULL;
|
||||
|
|
@ -1815,7 +1818,7 @@ prof_tdata_init_impl(tsdn_t *tsdn, uint64_t thr_uid, uint64_t thr_discrim,
|
|||
|
||||
if (ckh_new(tsdn, &tdata->bt2tctx, PROF_CKH_MINITEMS,
|
||||
prof_bt_hash, prof_bt_keycomp)) {
|
||||
idalloctm(tsdn, tdata, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(tdata), tdata, NULL, true, true);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
|
|
@ -1878,10 +1881,12 @@ prof_tdata_destroy_locked(tsdn_t *tsdn, prof_tdata_t *tdata,
|
|||
|
||||
assert(prof_tdata_should_destroy_unlocked(tdata, even_if_attached));
|
||||
|
||||
if (tdata->thread_name != NULL)
|
||||
idalloctm(tsdn, tdata->thread_name, NULL, true, true);
|
||||
if (tdata->thread_name != NULL) {
|
||||
idalloctm(tsdn, iealloc(tdata->thread_name), tdata->thread_name,
|
||||
NULL, true, true);
|
||||
}
|
||||
ckh_delete(tsdn, &tdata->bt2tctx);
|
||||
idalloctm(tsdn, tdata, NULL, true, true);
|
||||
idalloctm(tsdn, iealloc(tdata), tdata, NULL, true, true);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -2075,7 +2080,8 @@ prof_thread_name_set(tsd_t *tsd, const char *thread_name)
|
|||
return (EAGAIN);
|
||||
|
||||
if (tdata->thread_name != NULL) {
|
||||
idalloctm(tsd_tsdn(tsd), tdata->thread_name, NULL, true, true);
|
||||
idalloctm(tsd_tsdn(tsd), iealloc(tdata->thread_name),
|
||||
tdata->thread_name, NULL, true, true);
|
||||
tdata->thread_name = NULL;
|
||||
}
|
||||
if (strlen(s) > 0)
|
||||
|
|
|
|||
31
src/tcache.c
31
src/tcache.c
|
|
@ -27,7 +27,7 @@ size_t
|
|||
tcache_salloc(tsdn_t *tsdn, const void *ptr)
|
||||
{
|
||||
|
||||
return (arena_salloc(tsdn, ptr, false));
|
||||
return (arena_salloc(tsdn, iealloc(ptr), ptr, false));
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -101,9 +101,8 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
|
|||
assert(arena != NULL);
|
||||
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
|
||||
/* Lock the arena bin associated with the first object. */
|
||||
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
|
||||
*(tbin->avail - 1));
|
||||
arena_t *bin_arena = extent_arena_get(&chunk->extent);
|
||||
extent_t *extent = iealloc(*(tbin->avail - 1));
|
||||
arena_t *bin_arena = extent_arena_get(extent);
|
||||
arena_bin_t *bin = &bin_arena->bins[binind];
|
||||
|
||||
if (config_prof && bin_arena == arena) {
|
||||
|
|
@ -125,14 +124,17 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin,
|
|||
for (i = 0; i < nflush; i++) {
|
||||
ptr = *(tbin->avail - 1 - i);
|
||||
assert(ptr != NULL);
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
if (extent_arena_get(&chunk->extent) == bin_arena) {
|
||||
|
||||
extent = iealloc(ptr);
|
||||
if (extent_arena_get(extent) == bin_arena) {
|
||||
arena_chunk_t *chunk =
|
||||
(arena_chunk_t *)extent_addr_get(extent);
|
||||
size_t pageind = ((uintptr_t)ptr -
|
||||
(uintptr_t)chunk) >> LG_PAGE;
|
||||
arena_chunk_map_bits_t *bitselm =
|
||||
arena_bitselm_get_mutable(chunk, pageind);
|
||||
arena_dalloc_bin_junked_locked(tsd_tsdn(tsd),
|
||||
bin_arena, chunk, ptr, bitselm);
|
||||
bin_arena, chunk, extent, ptr, bitselm);
|
||||
} else {
|
||||
/*
|
||||
* This object was allocated via a different
|
||||
|
|
@ -183,9 +185,8 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
|||
assert(arena != NULL);
|
||||
for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
|
||||
/* Lock the arena associated with the first object. */
|
||||
arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
|
||||
*(tbin->avail - 1));
|
||||
arena_t *locked_arena = extent_arena_get(&chunk->extent);
|
||||
extent_t *extent = iealloc(*(tbin->avail - 1));
|
||||
arena_t *locked_arena = extent_arena_get(extent);
|
||||
UNUSED bool idump;
|
||||
|
||||
if (config_prof)
|
||||
|
|
@ -210,10 +211,12 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind,
|
|||
for (i = 0; i < nflush; i++) {
|
||||
ptr = *(tbin->avail - 1 - i);
|
||||
assert(ptr != NULL);
|
||||
chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
|
||||
if (extent_arena_get(&chunk->extent) == locked_arena) {
|
||||
extent = iealloc(ptr);
|
||||
if (extent_arena_get(extent) == locked_arena) {
|
||||
arena_chunk_t *chunk =
|
||||
(arena_chunk_t *)extent_addr_get(extent);
|
||||
arena_dalloc_large_junked_locked(tsd_tsdn(tsd),
|
||||
locked_arena, chunk, ptr);
|
||||
locked_arena, chunk, extent, ptr);
|
||||
} else {
|
||||
/*
|
||||
* This object was allocated via a different
|
||||
|
|
@ -391,7 +394,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache)
|
|||
arena_prof_accum(tsd_tsdn(tsd), arena, tcache->prof_accumbytes))
|
||||
prof_idump(tsd_tsdn(tsd));
|
||||
|
||||
idalloctm(tsd_tsdn(tsd), tcache, NULL, true, true);
|
||||
idalloctm(tsd_tsdn(tsd), iealloc(tcache), tcache, NULL, true, true);
|
||||
}
|
||||
|
||||
void
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue