romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-06-16 08:55:37 +03:00
Code Issues Projects Releases Packages Wiki Activity

Enable per-tcache tcache_max

Browse source 
1. add tcache_max and nhbins into tcache_t so that they are per-tcache,
   with one auto tcache per thread, it's also per-thread;
2. add mallctl for each thread to set its own tcache_max (of its auto tcache);
3. store the maximum number of items in each bin instead of using a global storage;
4. add tests for the modifications above.
5. Rename `nhbins` and `tcache_maxclass` to `global_do_not_change_nhbins` and `global_do_not_change_tcache_maxclass`.
This commit is contained in:
guangli-dai 2023-08-06 11:38:30 -07:00 • committed by Qi Wang
parent fbca96c433
commit a442d9b895
15 changed files with 528 additions and 222 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/arena.c
View file

@ -157,11 +157,18 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
cache_bin_array_descriptor_t *descriptor;
ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) {
for (szind_t i = 0; i < nhbins; i++) {
for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
cache_bin_t *cache_bin = &descriptor->bins[i];
cache_bin_sz_t ncached, nstashed;
cache_bin_nitems_get_remote(cache_bin,
&tcache_bin_info[i], &ncached, &nstashed);
&cache_bin->bin_info, &ncached, &nstashed);
if ((i < SC_NBINS &&
tcache_small_bin_disabled(i, cache_bin)) ||
(i >= SC_NBINS &&
tcache_large_bin_disabled(i, cache_bin))) {
assert(ncached == 0 && nstashed == 0);
}
astats->tcache_bytes += ncached * sz_index2size(i);
astats->tcache_stashed_bytes += nstashed *
@ -720,7 +727,8 @@ arena_dalloc_promoted_impl(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
safety_check_verify_redzone(ptr, usize, bumped_usize);
}
if (bumped_usize >= SC_LARGE_MINCLASS &&
bumped_usize <= tcache_maxclass && tcache != NULL) {
tcache != NULL &&
bumped_usize <= tcache_max_get(tcache)) {
tcache_dalloc_large(tsdn_tsd(tsdn), tcache, ptr,
sz_size2index(bumped_usize), slow_path);
} else {

1
src/cache_bin.c
View file

@ -82,6 +82,7 @@ cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, void *alloc,
bin->low_bits_low_water = (uint16_t)(uintptr_t)bin->stack_head;
bin->low_bits_full = (uint16_t)(uintptr_t)full_position;
bin->low_bits_empty = (uint16_t)(uintptr_t)empty_position;
cache_bin_info_init(&bin->bin_info, info->ncached_max);
cache_bin_sz_t free_spots = cache_bin_diff(bin,
bin->low_bits_full, (uint16_t)(uintptr_t)bin->stack_head);
assert(free_spots == bin_stack_size);

40
src/ctl.c
View file

@ -66,6 +66,7 @@ CTL_PROTO(epoch)
CTL_PROTO(background_thread)
CTL_PROTO(max_background_threads)
CTL_PROTO(thread_tcache_enabled)
CTL_PROTO(thread_tcache_max)
CTL_PROTO(thread_tcache_flush)
CTL_PROTO(thread_peak_read)
CTL_PROTO(thread_peak_reset)
@ -371,6 +372,7 @@ CTL_PROTO(stats_mutexes_reset)
static const ctl_named_node_t thread_tcache_node[] = {
{NAME("enabled"), CTL(thread_tcache_enabled)},
{NAME("max"), CTL(thread_tcache_max)},
{NAME("flush"), CTL(thread_tcache_flush)}
};
@ -2289,6 +2291,40 @@ label_return:
return ret;
}
static int
thread_tcache_max_ctl(tsd_t *tsd, const size_t *mib,
size_t miblen, void *oldp, size_t *oldlenp, void *newp,
size_t newlen) {
int ret;
size_t oldval;
/* pointer to tcache_t always exists even with tcache disabled. */
tcache_t *tcache = tsd_tcachep_get(tsd);
assert(tcache != NULL);
oldval = tcache_max_get(tcache);
READ(oldval, size_t);
if (newp != NULL) {
if (newlen != sizeof(size_t)) {
ret = EINVAL;
goto label_return;
}
size_t new_tcache_max = oldval;
WRITE(new_tcache_max, size_t);
if (new_tcache_max > TCACHE_MAXCLASS_LIMIT) {
new_tcache_max = TCACHE_MAXCLASS_LIMIT;
}
new_tcache_max = sz_s2u(new_tcache_max);
if(new_tcache_max != oldval) {
thread_tcache_max_and_nhbins_set(tsd, new_tcache_max);
}
}
ret = 0;
label_return:
return ret;
}
static int
thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib,
size_t miblen, void *oldp, size_t *oldlenp, void *newp,
@ -3101,9 +3137,9 @@ arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t)
CTL_RO_NL_GEN(arenas_tcache_max, global_do_not_change_tcache_maxclass, size_t)
CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned)
CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned)
CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_nhbins, unsigned)
CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t)

14
src/jemalloc.c
View file

@ -4136,15 +4136,13 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) {
filled += n;
}
if (likely(ind < nhbins) && progress < batch) {
unsigned tcache_ind = mallocx_tcache_get(flags);
tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind,
/* slow */ true, /* is_alloc */ true);
if (likely(tcache != NULL &&
ind < tcache_nhbins_get(tcache)) && progress < batch) {
if (bin == NULL) {
unsigned tcache_ind = mallocx_tcache_get(flags);
tcache_t *tcache = tcache_get_from_ind(tsd,
tcache_ind, /* slow */ true,
/* is_alloc */ true);
if (tcache != NULL) {
bin = &tcache->bins[ind];
}
bin = &tcache->bins[ind];
}
/*
* If we don't have a tcache bin, we don't want to

319
src/tcache.c
View file

@ -12,7 +12,7 @@
bool opt_tcache = true;
/* tcache_maxclass is set to 32KB by default. */
/* global_do_not_change_tcache_maxclass is set to 32KB by default. */
size_t opt_tcache_max = ((size_t)1) << 15;
/* Reasonable defaults for min and max values. */
@ -57,16 +57,18 @@ size_t opt_tcache_gc_delay_bytes = 0;
unsigned opt_lg_tcache_flush_small_div = 1;
unsigned opt_lg_tcache_flush_large_div = 1;
cache_bin_info_t *tcache_bin_info;
/* Total stack size required (per tcache). Include the padding above. */
static size_t tcache_bin_alloc_size;
static size_t tcache_bin_alloc_alignment;
/* Number of cache bins enabled, including both large and small. */
unsigned nhbins;
/* Max size class to be cached (can be small or large). */
size_t tcache_maxclass;
/*
* Number of cache bins enabled, including both large and small. This value
* is only used to initialize tcache_nhbins in the per-thread tcache.
* Directly modifying it will not affect threads already launched.
*/
unsigned global_do_not_change_nhbins;
/*
* Max size class to be cached (can be small or large). This value is only used
* to initialize tcache_max in the per-thread tcache. Directly modifying it
* will not affect threads already launched.
*/
size_t global_do_not_change_tcache_maxclass;
tcaches_t *tcaches;
@ -127,9 +129,9 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
cache_bin_t *cache_bin = &tcache->bins[szind];
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&tcache_bin_info[szind]);
&cache_bin->bin_info);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&tcache_bin_info[szind]);
&cache_bin->bin_info);
assert(!tcache_slow->bin_refilled[szind]);
size_t nflush = low_water - (low_water >> 2);
@ -152,7 +154,7 @@ tcache_gc_small(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
* Reduce fill count by 2X. Limit lg_fill_div such that
* the fill count is always at least 1.
*/
if ((cache_bin_info_ncached_max(&tcache_bin_info[szind])
if ((cache_bin_info_ncached_max(&cache_bin->bin_info)
>> (tcache_slow->lg_fill_div[szind] + 1)) >= 1) {
tcache_slow->lg_fill_div[szind]++;
}
@ -165,9 +167,9 @@ tcache_gc_large(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
assert(szind >= SC_NBINS);
cache_bin_t *cache_bin = &tcache->bins[szind];
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&tcache_bin_info[szind]);
&cache_bin->bin_info);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&tcache_bin_info[szind]);
&cache_bin->bin_info);
tcache_bin_flush_large(tsd, tcache, cache_bin, szind,
(unsigned)(ncached - low_water + (low_water >> 2)));
}
@ -187,7 +189,7 @@ tcache_event(tsd_t *tsd) {
tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&tcache_bin_info[szind]);
&cache_bin->bin_info);
if (low_water > 0) {
if (is_small) {
tcache_gc_small(tsd, tcache_slow, tcache, szind);
@ -208,7 +210,7 @@ tcache_event(tsd_t *tsd) {
cache_bin_low_water_set(cache_bin);
tcache_slow->next_gc_bin++;
if (tcache_slow->next_gc_bin == nhbins) {
if (tcache_slow->next_gc_bin == tcache_nhbins_get(tcache)) {
tcache_slow->next_gc_bin = 0;
}
}
@ -233,10 +235,10 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena,
void *ret;
assert(tcache_slow->arena != NULL);
unsigned nfill = cache_bin_info_ncached_max(&tcache_bin_info[binind])
unsigned nfill = cache_bin_info_ncached_max(&cache_bin->bin_info)
>> tcache_slow->lg_fill_div[binind];
arena_cache_bin_fill_small(tsdn, arena, cache_bin,
&tcache_bin_info[binind], binind, nfill);
&cache_bin->bin_info, binind, nfill);
tcache_slow->bin_refilled[binind] = true;
ret = cache_bin_alloc(cache_bin, tcache_success);
@ -318,7 +320,7 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
if (small) {
assert(binind < SC_NBINS);
} else {
assert(binind < nhbins);
assert(binind < tcache_nhbins_get(tcache));
}
arena_t *tcache_arena = tcache_slow->arena;
assert(tcache_arena != NULL);
@ -508,18 +510,18 @@ tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small);
cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin,
&tcache_bin_info[binind]);
&cache_bin->bin_info);
assert((cache_bin_sz_t)rem <= ncached);
unsigned nflush = ncached - rem;
CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush);
cache_bin_init_ptr_array_for_flush(cache_bin, &tcache_bin_info[binind],
cache_bin_init_ptr_array_for_flush(cache_bin, &cache_bin->bin_info,
&ptrs, nflush);
tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush,
small);
cache_bin_finish_flush(cache_bin, &tcache_bin_info[binind], &ptrs,
cache_bin_finish_flush(cache_bin, &cache_bin->bin_info, &ptrs,
ncached - rem);
}
@ -548,7 +550,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
void
tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
szind_t binind, bool is_small) {
cache_bin_info_t *info = &tcache_bin_info[binind];
cache_bin_info_t *info = &cache_bin->bin_info;
/*
* The two below are for assertion only. The content of original cached
* items remain unchanged -- the stashed items reside on the other end
@ -633,15 +635,31 @@ tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
tcache_arena_associate(tsdn, tcache_slow, tcache, arena);
}
static void
tcache_max_and_nhbins_init(tcache_t *tcache) {
assert(tcache != NULL);
assert(global_do_not_change_tcache_maxclass != 0);
assert(global_do_not_change_nhbins != 0);
tcache->tcache_max = global_do_not_change_tcache_maxclass;
tcache->tcache_nhbins = global_do_not_change_nhbins;
assert(tcache->tcache_nhbins == sz_size2index(tcache->tcache_max) + 1);
}
bool
tsd_tcache_enabled_data_init(tsd_t *tsd) {
/* Called upon tsd initialization. */
tsd_tcache_enabled_set(tsd, opt_tcache);
/*
* tcache is not available yet, but we need to set up its tcache_max
* and tcache_nhbins in advance.
*/
tcache_t *tcache = tsd_tcachep_get(tsd);
tcache_max_and_nhbins_init(tcache);
tsd_slow_update(tsd);
if (opt_tcache) {
/* Trigger tcache init. */
tsd_tcache_data_init(tsd);
tsd_tcache_data_init(tsd, NULL);
}
return false;
@ -649,7 +667,7 @@ tsd_tcache_enabled_data_init(tsd_t *tsd) {
static void
tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
void *mem) {
void *mem, cache_bin_info_t *tcache_bin_info) {
tcache->tcache_slow = tcache_slow;
tcache_slow->tcache = tcache;
@ -660,17 +678,19 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
/*
* We reserve cache bins for all small size classes, even if some may
* not get used (i.e. bins higher than nhbins). This allows the fast
* and common paths to access cache bin metadata safely w/o worrying
* about which ones are disabled.
* not get used (i.e. bins higher than tcache_nhbins). This allows
* the fast and common paths to access cache bin metadata safely w/o
* worrying about which ones are disabled.
*/
unsigned n_reserved_bins = nhbins < SC_NBINS ? SC_NBINS : nhbins;
unsigned tcache_nhbins = tcache_nhbins_get(tcache);
unsigned n_reserved_bins = tcache_nhbins < SC_NBINS ? SC_NBINS
: tcache_nhbins;
memset(tcache->bins, 0, sizeof(cache_bin_t) * n_reserved_bins);
size_t cur_offset = 0;
cache_bin_preincrement(tcache_bin_info, nhbins, mem,
cache_bin_preincrement(tcache_bin_info, tcache_nhbins, mem,
&cur_offset);
for (unsigned i = 0; i < nhbins; i++) {
for (unsigned i = 0; i < tcache_nhbins; i++) {
if (i < SC_NBINS) {
tcache_slow->lg_fill_div[i] = 1;
tcache_slow->bin_refilled[i] = false;
@ -682,12 +702,12 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
&cur_offset);
}
/*
* For small size classes beyond tcache_maxclass (i.e. nhbins < NBINS),
* their cache bins are initialized to a state to safely and efficiently
* fail all fastpath alloc / free, so that no additional check around
* nhbins is needed on fastpath.
* For small size classes beyond tcache_max(i.e.
* tcache_nhbins< NBINS), their cache bins are initialized to a state
* to safely and efficiently fail all fastpath alloc / free, so that
* no additional check around tcache_nhbins is needed on fastpath.
*/
for (unsigned i = nhbins; i < SC_NBINS; i++) {
for (unsigned i = tcache_nhbins; i < SC_NBINS; i++) {
/* Disabled small bins. */
cache_bin_t *cache_bin = &tcache->bins[i];
void *fake_stack = mem;
@ -699,19 +719,102 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache,
}
cache_bin_postincrement(mem, &cur_offset);
/* Sanity check that the whole stack is used. */
assert(cur_offset == tcache_bin_alloc_size);
if (config_debug) {
/* Sanity check that the whole stack is used. */
size_t size, alignment;
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins,
&size, &alignment);
assert(cur_offset == size);
}
}
static inline unsigned
tcache_ncached_max_compute(szind_t szind, unsigned current_nhbins) {
if (szind >= SC_NBINS) {
assert(szind < current_nhbins);
return opt_tcache_nslots_large;
}
unsigned slab_nregs = bin_infos[szind].nregs;
/* We may modify these values; start with the opt versions. */
unsigned nslots_small_min = opt_tcache_nslots_small_min;
unsigned nslots_small_max = opt_tcache_nslots_small_max;
/*
* Clamp values to meet our constraints -- even, nonzero, min < max, and
* suitable for a cache bin size.
*/
if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) {
nslots_small_max = CACHE_BIN_NCACHED_MAX;
}
if (nslots_small_min % 2 != 0) {
nslots_small_min++;
}
if (nslots_small_max % 2 != 0) {
nslots_small_max--;
}
if (nslots_small_min < 2) {
nslots_small_min = 2;
}
if (nslots_small_max < 2) {
nslots_small_max = 2;
}
if (nslots_small_min > nslots_small_max) {
nslots_small_min = nslots_small_max;
}
unsigned candidate;
if (opt_lg_tcache_nslots_mul < 0) {
candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul);
} else {
candidate = slab_nregs << opt_lg_tcache_nslots_mul;
}
if (candidate % 2 != 0) {
/*
* We need the candidate size to be even -- we assume that we
* can divide by two and get a positive number (e.g. when
* flushing).
*/
++candidate;
}
if (candidate <= nslots_small_min) {
return nslots_small_min;
} else if (candidate <= nslots_small_max) {
return candidate;
} else {
return nslots_small_max;
}
}
static void
tcache_bin_info_compute(cache_bin_info_t *tcache_bin_info,
unsigned tcache_nhbins) {
for (szind_t i = 0; i < tcache_nhbins; i++) {
unsigned ncached_max = tcache_ncached_max_compute(i,
tcache_nhbins);
cache_bin_info_init(&tcache_bin_info[i], ncached_max);
}
for (szind_t i = tcache_nhbins; i < SC_NBINS; i++) {
/* Disabled small bins. */
cache_bin_info_init(&tcache_bin_info[i], 0);
}
}
/* Initialize auto tcache (embedded in TSD). */
bool
tsd_tcache_data_init(tsd_t *tsd) {
tsd_tcache_data_init(tsd_t *tsd, arena_t *arena) {
tcache_slow_t *tcache_slow = tsd_tcache_slowp_get_unsafe(tsd);
tcache_t *tcache = tsd_tcachep_get_unsafe(tsd);
assert(cache_bin_still_zero_initialized(&tcache->bins[0]));
size_t alignment = tcache_bin_alloc_alignment;
size_t size = sz_sa2u(tcache_bin_alloc_size, alignment);
unsigned tcache_nhbins = tcache_nhbins_get(tcache);
size_t size, alignment;
/* Takes 146B stack space. */
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {0};
tcache_bin_info_compute(tcache_bin_info, tcache_nhbins);
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins,
&size, &alignment);
size = sz_sa2u(size, alignment);
void *mem = ipallocztm(tsd_tsdn(tsd), size, alignment, true, NULL,
true, arena_get(TSDN_NULL, 0, true));
@ -719,7 +822,7 @@ tsd_tcache_data_init(tsd_t *tsd) {
return true;
}
tcache_init(tsd, tcache_slow, tcache, mem);
tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info);
/*
* Initialization is a bit tricky here. After malloc init is done, all
* threads can rely on arena_choose and associate tcache accordingly.
@ -729,14 +832,15 @@ tsd_tcache_data_init(tsd_t *tsd) {
* arena_choose_hard() will re-associate properly.
*/
tcache_slow->arena = NULL;
arena_t *arena;
if (!malloc_initialized()) {
/* If in initialization, assign to a0. */
arena = arena_get(tsd_tsdn(tsd), 0, false);
tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache,
arena);
} else {
arena = arena_choose(tsd, NULL);
if (arena == NULL) {
arena = arena_choose(tsd, NULL);
}
/* This may happen if thread.tcache.enabled is used. */
if (tcache_slow->arena == NULL) {
tcache_arena_associate(tsd_tsdn(tsd), tcache_slow,
@ -756,21 +860,29 @@ tcache_create_explicit(tsd_t *tsd) {
* the beginning of the whole allocation (for freeing). The makes sure
* the cache bins have the requested alignment.
*/
size_t size = tcache_bin_alloc_size + sizeof(tcache_t)
unsigned tcache_nhbins = global_do_not_change_nhbins;
size_t tcache_size, alignment;
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {0};
tcache_bin_info_compute(tcache_bin_info, tcache_nhbins);
cache_bin_info_compute_alloc(tcache_bin_info, tcache_nhbins,
&tcache_size, &alignment);
size_t size = tcache_size + sizeof(tcache_t)
+ sizeof(tcache_slow_t);
/* Naturally align the pointer stacks. */
size = PTR_CEILING(size);
size = sz_sa2u(size, tcache_bin_alloc_alignment);
size = sz_sa2u(size, alignment);
void *mem = ipallocztm(tsd_tsdn(tsd), size, tcache_bin_alloc_alignment,
void *mem = ipallocztm(tsd_tsdn(tsd), size, alignment,
true, NULL, true, arena_get(TSDN_NULL, 0, true));
if (mem == NULL) {
return NULL;
}
tcache_t *tcache = (void *)((byte_t *)mem + tcache_bin_alloc_size);
tcache_t *tcache = (void *)((byte_t *)mem + tcache_size);
tcache_slow_t *tcache_slow =
(void *)((byte_t *)mem + tcache_bin_alloc_size + sizeof(tcache_t));
tcache_init(tsd, tcache_slow, tcache, mem);
(void *)((byte_t *)mem + tcache_size + sizeof(tcache_t));
tcache_max_and_nhbins_init(tcache);
tcache_init(tsd, tcache_slow, tcache, mem, tcache_bin_info);
tcache_arena_associate(tsd_tsdn(tsd), tcache_slow, tcache,
arena_ichoose(tsd, NULL));
@ -783,7 +895,7 @@ tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
tcache_slow_t *tcache_slow = tcache->tcache_slow;
assert(tcache_slow->arena != NULL);
for (unsigned i = 0; i < nhbins; i++) {
for (unsigned i = 0; i < tcache_nhbins_get(tcache); i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
if (i < SC_NBINS) {
tcache_bin_flush_small(tsd, tcache, cache_bin, i, 0);
@ -811,7 +923,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) {
if (tsd_tcache) {
cache_bin_t *cache_bin = &tcache->bins[0];
cache_bin_assert_empty(cache_bin, &tcache_bin_info[0]);
cache_bin_assert_empty(cache_bin, &cache_bin->bin_info);
}
idalloctm(tsd_tsdn(tsd), tcache_slow->dyn_alloc, NULL, NULL, true,
true);
@ -849,13 +961,9 @@ tcache_cleanup(tsd_t *tsd) {
assert(!cache_bin_still_zero_initialized(&tcache->bins[0]));
tcache_destroy(tsd, tcache, true);
if (config_debug) {
/*
* For debug testing only, we want to pretend we're still in the
* zero-initialized state.
*/
memset(tcache->bins, 0, sizeof(cache_bin_t) * nhbins);
}
/* Make sure all bins used are reinitialized to the clean state. */
memset(tcache->bins, 0, sizeof(cache_bin_t) *
tcache_nhbins_get(tcache));
}
void
@ -863,7 +971,7 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
cassert(config_stats);
/* Merge and reset tcache stats. */
for (unsigned i = 0; i < nhbins; i++) {
for (unsigned i = 0; i < tcache_nhbins_get(tcache); i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
if (i < SC_NBINS) {
bin_t *bin = arena_bin_choose(tsdn, arena, i, NULL);
@ -986,97 +1094,18 @@ tcaches_destroy(tsd_t *tsd, unsigned ind) {
}
}
static unsigned
tcache_ncached_max_compute(szind_t szind) {
if (szind >= SC_NBINS) {
assert(szind < nhbins);
return opt_tcache_nslots_large;
}
unsigned slab_nregs = bin_infos[szind].nregs;
/* We may modify these values; start with the opt versions. */
unsigned nslots_small_min = opt_tcache_nslots_small_min;
unsigned nslots_small_max = opt_tcache_nslots_small_max;
/*
* Clamp values to meet our constraints -- even, nonzero, min < max, and
* suitable for a cache bin size.
*/
if (opt_tcache_nslots_small_max > CACHE_BIN_NCACHED_MAX) {
nslots_small_max = CACHE_BIN_NCACHED_MAX;
}
if (nslots_small_min % 2 != 0) {
nslots_small_min++;
}
if (nslots_small_max % 2 != 0) {
nslots_small_max--;
}
if (nslots_small_min < 2) {
nslots_small_min = 2;
}
if (nslots_small_max < 2) {
nslots_small_max = 2;
}
if (nslots_small_min > nslots_small_max) {
nslots_small_min = nslots_small_max;
}
unsigned candidate;
if (opt_lg_tcache_nslots_mul < 0) {
candidate = slab_nregs >> (-opt_lg_tcache_nslots_mul);
} else {
candidate = slab_nregs << opt_lg_tcache_nslots_mul;
}
if (candidate % 2 != 0) {
/*
* We need the candidate size to be even -- we assume that we
* can divide by two and get a positive number (e.g. when
* flushing).
*/
++candidate;
}
if (candidate <= nslots_small_min) {
return nslots_small_min;
} else if (candidate <= nslots_small_max) {
return candidate;
} else {
return nslots_small_max;
}
}
bool
tcache_boot(tsdn_t *tsdn, base_t *base) {
tcache_maxclass = sz_s2u(opt_tcache_max);
assert(tcache_maxclass <= TCACHE_MAXCLASS_LIMIT);
nhbins = sz_size2index(tcache_maxclass) + 1;
global_do_not_change_tcache_maxclass = sz_s2u(opt_tcache_max);
assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT);
global_do_not_change_nhbins =
sz_size2index(global_do_not_change_tcache_maxclass) + 1;
if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,
malloc_mutex_rank_exclusive)) {
return true;
}
/* Initialize tcache_bin_info. See comments in tcache_init(). */
unsigned n_reserved_bins = nhbins < SC_NBINS ? SC_NBINS : nhbins;
size_t size = n_reserved_bins * sizeof(cache_bin_info_t);
tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, base, size,
CACHELINE);
if (tcache_bin_info == NULL) {
return true;
}
for (szind_t i = 0; i < nhbins; i++) {
unsigned ncached_max = tcache_ncached_max_compute(i);
cache_bin_info_init(&tcache_bin_info[i], ncached_max);
}
for (szind_t i = nhbins; i < SC_NBINS; i++) {
/* Disabled small bins. */
cache_bin_info_init(&tcache_bin_info[i], 0);
assert(tcache_small_bin_disabled(i, NULL));
}
cache_bin_info_compute_alloc(tcache_bin_info, nhbins,
&tcache_bin_alloc_size, &tcache_bin_alloc_alignment);
return false;
}