diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index 9b7c05f9..4dae64ce 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -129,6 +129,7 @@ void arena_prefork6(tsdn_t *tsdn, arena_t *arena); void arena_prefork7(tsdn_t *tsdn, arena_t *arena); void arena_prefork8(tsdn_t *tsdn, arena_t *arena); void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena); -void arena_postfork_child(tsdn_t *tsdn, arena_t *arena); +void arena_postfork_child(tsdn_t *tsdn, arena_t *arena, + cache_bin_array_descriptor_t *surviving_desc); #endif /* JEMALLOC_INTERNAL_ARENA_EXTERNS_H */ diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 21b94e2e..4dc0bae9 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -72,7 +72,8 @@ void tcaches_destroy(tsd_t *tsd, unsigned ind); bool tcache_boot(tsdn_t *tsdn, base_t *base); void tcache_arena_associate( tsdn_t *tsdn, tcache_slow_t *tcache_slow, arena_t *arena); -void tcache_arena_postfork_child(tsdn_t *tsdn, arena_t *arena); +cache_bin_array_descriptor_t *tcache_postfork_arena_descriptor( + tsdn_t *tsdn, arena_t *arena); void tcache_prefork(tsdn_t *tsdn); void tcache_postfork_parent(tsdn_t *tsdn); void tcache_postfork_child(tsdn_t *tsdn); diff --git a/src/arena.c b/src/arena.c index 3831b49c..23292d0f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -271,10 +271,9 @@ arena_cache_bin_array_unregister(tsdn_t *tsdn, arena_t *arena, } /* - * Postfork-child entry: child is single-threaded and the queue is rebuilt - * from scratch (descriptors held by other threads at fork time are gone). - * The mutex itself is reinitialized later in arena_postfork_child, so we - * cannot lock here. + * Postfork-child entry: child is single-threaded, so the queue is rebuilt + * from scratch (descriptors held by other threads at fork time are gone) + * without locking. */ void arena_cache_bin_array_postfork_child(arena_t *arena, @@ -2160,7 +2159,8 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { } void -arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { +arena_postfork_child(tsdn_t *tsdn, arena_t *arena, + cache_bin_array_descriptor_t *surviving_desc) { atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED); atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED); if (tsd_arena_get(tsdn_tsd(tsdn)) == arena) { @@ -2169,7 +2169,11 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { if (tsd_iarena_get(tsdn_tsd(tsdn)) == arena) { arena_nthreads_inc(arena, true); } - tcache_arena_postfork_child(tsdn, arena); + if (config_stats) { + malloc_mutex_postfork_child(tsdn, + &arena->cache_bin_array_descriptor_ql_mtx); + arena_cache_bin_array_postfork_child(arena, surviving_desc); + } for (unsigned i = 0; i < nbins_total; i++) { JEMALLOC_SUPPRESS_WARN_ON_USAGE( @@ -2179,7 +2183,4 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_postfork_child(tsdn, &arena->large_mtx); base_postfork_child(tsdn, arena->base); pa_shard_postfork_child(tsdn, &arena->pa_shard); - if (config_stats) { - malloc_mutex_postfork_child(tsdn, &arena->cache_bin_array_descriptor_ql_mtx); - } } diff --git a/src/jemalloc_fork.c b/src/jemalloc_fork.c index bd41383c..11d246de 100644 --- a/src/jemalloc_fork.c +++ b/src/jemalloc_fork.c @@ -150,7 +150,10 @@ jemalloc_postfork_child(void) { arena_t *arena; if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { - arena_postfork_child(tsd_tsdn(tsd), arena); + cache_bin_array_descriptor_t *desc = + tcache_postfork_arena_descriptor( + tsd_tsdn(tsd), arena); + arena_postfork_child(tsd_tsdn(tsd), arena, desc); } } prof_postfork_child(tsd_tsdn(tsd)); diff --git a/src/tcache.c b/src/tcache.c index 0cba2ebe..8c2f6f4c 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -754,21 +754,20 @@ tcache_arena_reassociate(tsdn_t *tsdn, tcache_slow_t *tcache_slow, tcache_arena_associate(tsdn, tcache_slow, arena); } -void -tcache_arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { +cache_bin_array_descriptor_t * +tcache_postfork_arena_descriptor(tsdn_t *tsdn, arena_t *arena) { if (!config_stats) { - return; + return NULL; } - cache_bin_array_descriptor_t *desc = NULL; tcache_slow_t *tcache_slow = tcache_slow_get(tsdn_tsd(tsdn)); - if (tcache_slow != NULL && tcache_slow->arena == arena) { - assert(tcache_slow->tcache != NULL); - cache_bin_array_descriptor_init( - &tcache_slow->cache_bin_array_descriptor, - tcache_slow->tcache->bins); - desc = &tcache_slow->cache_bin_array_descriptor; + if (tcache_slow == NULL || tcache_slow->arena != arena) { + return NULL; } - arena_cache_bin_array_postfork_child(arena, desc); + assert(tcache_slow->tcache != NULL); + cache_bin_array_descriptor_init( + &tcache_slow->cache_bin_array_descriptor, + tcache_slow->tcache->bins); + return &tcache_slow->cache_bin_array_descriptor; } static void