romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-05-28 15:47:31 +03:00
Code Issues Projects Releases Packages Wiki Activity

Replace arena->tcache_ql with cache_bin_array_descriptor_ql walks

Browse source 
Drop the duplicate arena->tcache_ql; stats merging walks the
cache_bin_array_descriptor_ql directly. Rename the protecting mutex
from tcache_ql_mtx to cache_bin_array_descriptor_ql_mtx to match. Add
an assertion in test_thread_migrate_arena that the dissociate-time
flush zeros cache_bin->tstats.nrequests.
This commit is contained in:
Slobodan Predolac 2026-05-08 10:01:00 -07:00
parent f9c84860e0
commit b92420d309
9 changed files with 122 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/arena.c
View file

@ -164,7 +164,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
/* Currently cached bytes and sanitizer-stashed bytes in tcache. */
astats->tcache_bytes = 0;
astats->tcache_stashed_bytes = 0;
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
cache_bin_array_descriptor_t *descriptor;
ql_foreach (descriptor, &arena->cache_bin_array_descriptor_ql, link) {
for (szind_t i = 0; i < TCACHE_NBINS_MAX; i++) {
@ -183,8 +183,8 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
}
malloc_mutex_prof_read(tsdn,
&astats->mutex_prof_data[arena_prof_mutex_tcache_list],
&arena->tcache_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
&arena->cache_bin_array_descriptor_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
#define READ_ARENA_MUTEX_PROF_DATA(mtx, ind) \
malloc_mutex_lock(tsdn, &arena->mtx); \
@ -1809,10 +1809,11 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
goto label_error;
}
ql_new(&arena->tcache_ql);
ql_new(&arena->cache_bin_array_descriptor_ql);
if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql",
WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) {
if (malloc_mutex_init(&arena->cache_bin_array_descriptor_ql_mtx,
"cache_bin_array_descriptor_ql",
WITNESS_RANK_CACHE_BIN_ARRAY_DESCRIPTOR_QL,
malloc_mutex_rank_exclusive)) {
goto label_error;
}
}
@ -2022,7 +2023,7 @@ arena_prefork0(tsdn_t *tsdn, arena_t *arena) {
void
arena_prefork1(tsdn_t *tsdn, arena_t *arena) {
if (config_stats) {
malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx);
malloc_mutex_prefork(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
}
@ -2075,7 +2076,7 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
base_postfork_parent(tsdn, arena->base);
pa_shard_postfork_parent(tsdn, &arena->pa_shard);
if (config_stats) {
malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx);
malloc_mutex_postfork_parent(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
}
@ -2100,6 +2101,6 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
base_postfork_child(tsdn, arena->base);
pa_shard_postfork_child(tsdn, &arena->pa_shard);
if (config_stats) {
malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx);
malloc_mutex_postfork_child(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
}

2
src/ctl.c
View file

@ -3974,7 +3974,7 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
MUTEX_PROF_RESET(arena->pa_shard.pac.ecache_pinned.mtx);
MUTEX_PROF_RESET(arena->pa_shard.pac.decay_dirty.mtx);
MUTEX_PROF_RESET(arena->pa_shard.pac.decay_muzzy.mtx);
MUTEX_PROF_RESET(arena->tcache_ql_mtx);
MUTEX_PROF_RESET(arena->cache_bin_array_descriptor_ql_mtx);
MUTEX_PROF_RESET(arena->base->mtx);
for (szind_t j = 0; j < SC_NBINS; j++) {

14
src/jemalloc_init.c
View file

@ -305,16 +305,16 @@ stats_print_atexit(void) {
for (i = 0, narenas = narenas_total_get(); i < narenas; i++) {
arena_t *arena = arena_get(tsdn, i, false);
if (arena != NULL) {
tcache_slow_t *tcache_slow;
cache_bin_array_descriptor_t *desc;
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
ql_foreach (
tcache_slow, &arena->tcache_ql, link) {
tcache_stats_merge(
tsdn, tcache_slow->tcache, arena);
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
ql_foreach (desc,
&arena->cache_bin_array_descriptor_ql,
link) {
tcache_stats_merge(tsdn, desc, arena);
}
malloc_mutex_unlock(
tsdn, &arena->tcache_ql_mtx);
tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
}
}

42
src/tcache.c
View file

@ -721,15 +721,12 @@ tcache_bin_ncached_max_read(
}
/*
* Insert tcache_slow into arena's tcache list. Caller must hold
* arena->tcache_ql_mtx, OR be in the postfork-child path (single-threaded,
* mutex re-init pending). config_stats must be true.
* Caller must hold arena->cache_bin_array_descriptor_ql_mtx, OR be in the postfork-child path
* (single-threaded, mutex re-init pending).
*/
static void
tcache_arena_link(arena_t *arena, tcache_slow_t *tcache_slow,
tcache_t *tcache) {
ql_elm_new(tcache_slow, link);
ql_tail_insert(&arena->tcache_ql, tcache_slow, link);
cache_bin_array_descriptor_init(
&tcache_slow->cache_bin_array_descriptor, tcache->bins);
ql_tail_insert(&arena->cache_bin_array_descriptor_ql,
@ -743,9 +740,9 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
tcache_slow->arena = arena;
if (config_stats) {
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
tcache_arena_link(arena, tcache_slow, tcache);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
}
@ -755,24 +752,25 @@ tcache_arena_dissociate(
arena_t *arena = tcache_slow->arena;
assert(arena != NULL);
if (config_stats) {
/* Unlink from list of extant tcaches. */
malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx);
malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
if (config_debug) {
bool in_ql = false;
tcache_slow_t *iter;
ql_foreach (iter, &arena->tcache_ql, link) {
if (iter == tcache_slow) {
bool in_ql = false;
cache_bin_array_descriptor_t *iter;
ql_foreach (iter,
&arena->cache_bin_array_descriptor_ql, link) {
if (iter ==
&tcache_slow->cache_bin_array_descriptor) {
in_ql = true;
break;
}
}
assert(in_ql);
}
ql_remove(&arena->tcache_ql, tcache_slow, link);
ql_remove(&arena->cache_bin_array_descriptor_ql,
&tcache_slow->cache_bin_array_descriptor, link);
tcache_stats_merge(tsdn, tcache_slow->tcache, arena);
malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx);
tcache_stats_merge(tsdn,
&tcache_slow->cache_bin_array_descriptor, arena);
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
}
tcache_slow->arena = NULL;
}
@ -789,7 +787,6 @@ tcache_arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
if (!config_stats) {
return;
}
ql_new(&arena->tcache_ql);
ql_new(&arena->cache_bin_array_descriptor_ql);
tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn));
if (tcache_slow != NULL && tcache_slow->arena == arena) {
@ -811,7 +808,6 @@ tcache_init(tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, void *mem,
tcache->tcache_slow = tcache_slow;
tcache_slow->tcache = tcache;
memset(&tcache_slow->link, 0, sizeof(ql_elm(tcache_t)));
nstime_init_zero(&tcache_slow->last_gc_time);
tcache_slow->next_gc_bin = 0;
tcache_slow->next_gc_bin_small = 0;
@ -1277,13 +1273,13 @@ tcache_cleanup(tsd_t *tsd) {
}
void
tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) {
tcache_stats_merge(tsdn_t *tsdn, cache_bin_array_descriptor_t *desc,
arena_t *arena) {
cassert(config_stats);
/* Merge and reset tcache stats. */
for (unsigned i = 0; i < tcache_nbins_get(tcache->tcache_slow); i++) {
cache_bin_t *cache_bin = &tcache->bins[i];
if (tcache_bin_disabled(i, cache_bin, tcache->tcache_slow)) {
for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) {
cache_bin_t *cache_bin = &desc->bins[i];
if (cache_bin_disabled(cache_bin)) {
continue;
}
if (i < SC_NBINS) {