diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index 16bdb295..9b7c05f9 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -118,6 +118,7 @@ void arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena, cache_bin_array_descriptor_t *desc); void arena_cache_bin_array_postfork_child(arena_t *arena, cache_bin_array_descriptor_t *desc_or_null); +void arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index cc86b471..21b94e2e 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -66,8 +66,6 @@ void tcache_arena_reassociate( tcache_t *tcache_create_explicit(tsd_t *tsd); bool thread_tcache_max_set(tsd_t *tsd, size_t tcache_max); void tcache_cleanup(tsd_t *tsd); -void tcache_stats_merge(tsdn_t *tsdn, - cache_bin_array_descriptor_t *desc, arena_t *arena); bool tcaches_create(tsd_t *tsd, base_t *base, unsigned *r_ind); void tcaches_flush(tsd_t *tsd, unsigned ind); void tcaches_destroy(tsd_t *tsd, unsigned ind); diff --git a/src/arena.c b/src/arena.c index 3df4b3e1..3831b49c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -228,6 +228,27 @@ arena_cache_bin_array_register(tsdn_t *tsdn, arena_t *arena, malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx); } +static void +arena_cache_bin_stats_flush(tsdn_t *tsdn, arena_t *arena, + cache_bin_array_descriptor_t *desc) { + cassert(config_stats); + for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) { + cache_bin_t *cache_bin = &desc->bins[i]; + if (cache_bin_disabled(cache_bin)) { + continue; + } + if (i < SC_NBINS) { + bin_t *bin = bin_choose(tsdn, arena, i, NULL); + bin_stats_nrequests_add(tsdn, bin, + cache_bin->tstats.nrequests); + } else { + arena_stats_large_flush_nrequests_add(tsdn, + &arena->stats, i, cache_bin->tstats.nrequests); + } + cache_bin->tstats.nrequests = 0; + } +} + void arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena, cache_bin_array_descriptor_t *desc) { @@ -245,13 +266,7 @@ arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena, assert(in_ql); } ql_remove(&arena->cache_bin_array_descriptor_ql, desc, link); - /* - * Flush this descriptor's per-cache_bin request counts up to the - * arena's bin/large stats before the owner forgets which arena - * accumulated them. (Step 2 of this refactor will relocate - * tcache_stats_merge into cache_bin.c.) - */ - tcache_stats_merge(tsdn, desc, arena); + arena_cache_bin_stats_flush(tsdn, arena, desc); malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx); } @@ -272,6 +287,17 @@ arena_cache_bin_array_postfork_child(arena_t *arena, } } +void +arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) { + cassert(config_stats); + malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx); + cache_bin_array_descriptor_t *desc; + ql_foreach (desc, &arena->cache_bin_array_descriptor_ql, link) { + arena_cache_bin_stats_flush(tsdn, arena, desc); + } + malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx); +} + static void arena_background_thread_inactivity_check( tsdn_t *tsdn, arena_t *arena, bool is_background_thread) { diff --git a/src/jemalloc_init.c b/src/jemalloc_init.c index 72b204bd..b6b66643 100644 --- a/src/jemalloc_init.c +++ b/src/jemalloc_init.c @@ -305,16 +305,7 @@ stats_print_atexit(void) { for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { arena_t *arena = arena_get(tsdn, i, false); if (arena != NULL) { - cache_bin_array_descriptor_t *desc; - - malloc_mutex_lock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx); - ql_foreach (desc, - &arena->cache_bin_array_descriptor_ql, - link) { - tcache_stats_merge(tsdn, desc, arena); - } - malloc_mutex_unlock( - tsdn, &arena->cache_bin_array_descriptor_ql_mtx); + arena_cache_bins_stats_merge(tsdn, arena); } } } diff --git a/src/tcache.c b/src/tcache.c index e7eef6c0..0cba2ebe 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -1248,28 +1248,6 @@ tcache_cleanup(tsd_t *tsd) { memset(tcache->bins, 0, sizeof(cache_bin_t) * TCACHE_NBINS_MAX); } -void -tcache_stats_merge(tsdn_t *tsdn, cache_bin_array_descriptor_t *desc, - arena_t *arena) { - cassert(config_stats); - - for (unsigned i = 0; i < TCACHE_NBINS_MAX; i++) { - cache_bin_t *cache_bin = &desc->bins[i]; - if (cache_bin_disabled(cache_bin)) { - continue; - } - if (i < SC_NBINS) { - bin_t *bin = bin_choose(tsdn, arena, i, NULL); - bin_stats_nrequests_add(tsdn, bin, - cache_bin->tstats.nrequests); - } else { - arena_stats_large_flush_nrequests_add(tsdn, - &arena->stats, i, cache_bin->tstats.nrequests); - } - cache_bin->tstats.nrequests = 0; - } -} - static bool tcaches_create_prep(tsd_t *tsd, base_t *base) { bool err;