From ab25d3c987ddb32846760cc08af8db22a6389c02 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 12 Feb 2017 18:50:53 -0800 Subject: [PATCH] Synchronize arena->tcache_ql with arena->tcache_ql_mtx. This replaces arena->lock synchronization. --- include/jemalloc/internal/arena_structs_b.h | 3 ++- include/jemalloc/internal/witness_types.h | 13 +++++++------ src/arena.c | 15 +++++++++++++++ src/jemalloc.c | 11 +++-------- src/tcache.c | 10 ++++------ 5 files changed, 31 insertions(+), 21 deletions(-) diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h index 04e859b5..132a328b 100644 --- a/include/jemalloc/internal/arena_structs_b.h +++ b/include/jemalloc/internal/arena_structs_b.h @@ -135,9 +135,10 @@ struct arena_s { * Stats from these are merged incrementally, and at exit if * opt_stats_print is enabled. * - * Synchronization: lock. + * Synchronization: tcache_ql_mtx. */ ql_head(tcache_t) tcache_ql; + malloc_mutex_t tcache_ql_mtx; /* Synchronization: internal. */ prof_accum_t prof_accum; diff --git a/include/jemalloc/internal/witness_types.h b/include/jemalloc/internal/witness_types.h index 3fd7998a..7957b410 100644 --- a/include/jemalloc/internal/witness_types.h +++ b/include/jemalloc/internal/witness_types.h @@ -35,13 +35,14 @@ typedef int witness_comp_t (const witness_t *, void *, const witness_t *, #define WITNESS_RANK_CORE 9U #define WITNESS_RANK_ARENA 9U -#define WITNESS_RANK_EXTENTS 10U -#define WITNESS_RANK_EXTENT_FREELIST 11U +#define WITNESS_RANK_TCACHE_QL 10U +#define WITNESS_RANK_EXTENTS 11U +#define WITNESS_RANK_EXTENT_FREELIST 12U -#define WITNESS_RANK_RTREE_ELM 12U -#define WITNESS_RANK_RTREE 13U -#define WITNESS_RANK_BASE 14U -#define WITNESS_RANK_ARENA_LARGE 15U +#define WITNESS_RANK_RTREE_ELM 13U +#define WITNESS_RANK_RTREE 14U +#define WITNESS_RANK_BASE 15U +#define WITNESS_RANK_ARENA_LARGE 16U #define WITNESS_RANK_LEAF 0xffffffffU #define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF diff --git a/src/arena.c b/src/arena.c index ac447199..f4e051ca 100644 --- a/src/arena.c +++ b/src/arena.c @@ -215,6 +215,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, /* tcache_bytes counts currently cached bytes. */ astats->tcache_bytes = 0; + malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); ql_foreach(tcache, &arena->tcache_ql, link) { for (i = 0; i < nhbins; i++) { tbin = &tcache->tbins[i]; @@ -222,6 +223,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, index2size(i); } } + malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } for (i = 0; i < NBINS; i++) { @@ -1650,6 +1652,10 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { if (config_stats && config_tcache) { ql_new(&arena->tcache_ql); + if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql", + WITNESS_RANK_TCACHE_QL)) { + goto label_error; + } } if (config_prof) { @@ -1736,6 +1742,9 @@ arena_boot(void) { void arena_prefork0(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_prefork(tsdn, &arena->lock); + if (config_stats && config_tcache) { + malloc_mutex_prefork(tsdn, &arena->tcache_ql_mtx); + } } void @@ -1773,6 +1782,9 @@ arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { extents_postfork_parent(tsdn, &arena->extents_cached); extents_postfork_parent(tsdn, &arena->extents_retained); malloc_mutex_postfork_parent(tsdn, &arena->lock); + if (config_stats && config_tcache) { + malloc_mutex_postfork_parent(tsdn, &arena->tcache_ql_mtx); + } } void @@ -1788,4 +1800,7 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { extents_postfork_child(tsdn, &arena->extents_cached); extents_postfork_child(tsdn, &arena->extents_retained); malloc_mutex_postfork_child(tsdn, &arena->lock); + if (config_stats && config_tcache) { + malloc_mutex_postfork_child(tsdn, &arena->tcache_ql_mtx); + } } diff --git a/src/jemalloc.c b/src/jemalloc.c index d2c33bbc..197f9bdc 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -684,17 +684,12 @@ stats_print_atexit(void) { if (arena != NULL) { tcache_t *tcache; - /* - * tcache_stats_merge() locks bins, so if any - * code is introduced that acquires both arena - * and bin locks in the opposite order, - * deadlocks may result. - */ - malloc_mutex_lock(tsdn, &arena->lock); + malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); ql_foreach(tcache, &arena->tcache_ql, link) { tcache_stats_merge(tsdn, tcache, arena); } - malloc_mutex_unlock(tsdn, &arena->lock); + malloc_mutex_unlock(tsdn, + &arena->tcache_ql_mtx); } } } diff --git a/src/tcache.c b/src/tcache.c index 075f3481..dff31d19 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -261,10 +261,10 @@ static void tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { if (config_stats) { /* Link into list of extant tcaches. */ - malloc_mutex_lock(tsdn, &arena->lock); + malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); ql_elm_new(tcache, link); ql_tail_insert(&arena->tcache_ql, tcache, link); - malloc_mutex_unlock(tsdn, &arena->lock); + malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } } @@ -272,7 +272,7 @@ static void tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { if (config_stats) { /* Unlink from list of extant tcaches. */ - malloc_mutex_lock(tsdn, &arena->lock); + malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); if (config_debug) { bool in_ql = false; tcache_t *iter; @@ -286,7 +286,7 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { } ql_remove(&arena->tcache_ql, tcache, link); tcache_stats_merge(tsdn, tcache, arena); - malloc_mutex_unlock(tsdn, &arena->lock); + malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } } @@ -409,8 +409,6 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { cassert(config_stats); - malloc_mutex_assert_owner(tsdn, &arena->lock); - /* Merge and reset tcache stats. */ for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i];