From 589638182ae58ae8031eac2cd9ba9d5b05783b42 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Fri, 30 Oct 2020 14:43:43 -0700 Subject: [PATCH] Use the edata_cache_small_t in the HPA. --- include/jemalloc/internal/hpa.h | 11 ++++++++++- include/jemalloc/internal/hpa_central.h | 2 +- src/hpa.c | 23 ++++++++++++++++++----- src/hpa_central.c | 10 +++++----- src/pa.c | 3 ++- 5 files changed, 36 insertions(+), 13 deletions(-) diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h index 24c68560..1cef6e5d 100644 --- a/include/jemalloc/internal/hpa.h +++ b/include/jemalloc/internal/hpa.h @@ -24,6 +24,9 @@ struct hpa_s { /* * This edata cache is the global one that we use for new allocations in * growing; practically, it comes from a0. + * + * We don't use an edata_cache_small in front of this, since we expect a + * small finite number of allocations from it. */ edata_cache_t *edata_cache; geom_grow_t geom_grow; @@ -50,7 +53,7 @@ struct hpa_shard_s { * from a pageslab. The pageslab itself comes from the centralized * allocator, and so will use its edata_cache. */ - edata_cache_t *edata_cache; + edata_cache_small_t ecs; hpa_t *hpa; psset_t psset; @@ -86,6 +89,12 @@ bool hpa_init(hpa_t *hpa, base_t *base, emap_t *emap, bool hpa_shard_init(hpa_shard_t *shard, hpa_t *hpa, edata_cache_t *edata_cache, unsigned ind, size_t ps_goal, size_t ps_alloc_max, size_t small_max, size_t large_min); +/* + * Notify the shard that we won't use it for allocations much longer. Due to + * the possibility of races, we don't actually prevent allocations; just flush + * and disable the embedded edata_cache_small. + */ +void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard); void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard); /* diff --git a/include/jemalloc/internal/hpa_central.h b/include/jemalloc/internal/hpa_central.h index b90ca41e..8659f712 100644 --- a/include/jemalloc/internal/hpa_central.h +++ b/include/jemalloc/internal/hpa_central.h @@ -9,7 +9,7 @@ struct hpa_central_s { /* The emap we use for metadata operations. */ emap_t *emap; - edata_cache_t *edata_cache; + edata_cache_small_t ecs; eset_t eset; size_t sn_next; diff --git a/src/hpa.c b/src/hpa.c index f49aa2b0..b329dbbb 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -64,7 +64,8 @@ hpa_shard_init(hpa_shard_t *shard, hpa_t *hpa, edata_cache_t *edata_cache, return true; } - shard->edata_cache = edata_cache; + assert(edata_cache != NULL); + edata_cache_small_init(&shard->ecs, edata_cache); shard->hpa = hpa; psset_init(&shard->psset); shard->ps_goal = ps_goal; @@ -201,13 +202,14 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) { assert(size <= shard->ps_alloc_max); bool err; - edata_t *edata = edata_cache_get(tsdn, shard->edata_cache); + malloc_mutex_lock(tsdn, &shard->mtx); + edata_t *edata = edata_cache_small_get(tsdn, &shard->ecs); if (edata == NULL) { + malloc_mutex_unlock(tsdn, &shard->mtx); return NULL; } edata_arena_ind_set(edata, shard->ind); - malloc_mutex_lock(tsdn, &shard->mtx); err = psset_alloc_reuse(&shard->psset, edata, size); malloc_mutex_unlock(tsdn, &shard->mtx); if (!err) { @@ -229,7 +231,11 @@ hpa_alloc_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size) { shard->ps_goal); if (grow_edata == NULL) { malloc_mutex_unlock(tsdn, &shard->grow_mtx); - edata_cache_put(tsdn, shard->edata_cache, edata); + + malloc_mutex_lock(tsdn, &shard->mtx); + edata_cache_small_put(tsdn, &shard->ecs, edata); + malloc_mutex_unlock(tsdn, &shard->mtx); + return NULL; } edata_arena_ind_set(grow_edata, shard->ind); @@ -351,9 +357,9 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { malloc_mutex_lock(tsdn, &shard->mtx); edata_t *evicted_ps = psset_dalloc(&shard->psset, edata); + edata_cache_small_put(tsdn, &shard->ecs, edata); malloc_mutex_unlock(tsdn, &shard->mtx); - edata_cache_put(tsdn, shard->edata_cache, edata); if (evicted_ps != NULL) { /* @@ -387,6 +393,13 @@ hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) { assert(bin_stats->ninactive == 0); } +void +hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) { + malloc_mutex_lock(tsdn, &shard->mtx); + edata_cache_small_disable(tsdn, &shard->ecs); + malloc_mutex_unlock(tsdn, &shard->mtx); +} + void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) { /* diff --git a/src/hpa_central.c b/src/hpa_central.c index a1895c87..346d9422 100644 --- a/src/hpa_central.c +++ b/src/hpa_central.c @@ -7,7 +7,7 @@ void hpa_central_init(hpa_central_t *central, edata_cache_t *edata_cache, emap_t *emap) { central->emap = emap; - central->edata_cache = edata_cache; + edata_cache_small_init(¢ral->ecs, edata_cache); eset_init(¢ral->eset, extent_state_dirty); central->sn_next = 0; } @@ -19,7 +19,7 @@ hpa_central_init(hpa_central_t *central, edata_cache_t *edata_cache, static edata_t * hpa_central_split(tsdn_t *tsdn, hpa_central_t *central, edata_t *edata, size_t size) { - edata_t *trail = edata_cache_get(tsdn, central->edata_cache); + edata_t *trail = edata_cache_small_get(tsdn, ¢ral->ecs); if (trail == NULL) { return NULL; } @@ -34,7 +34,7 @@ hpa_central_split(tsdn_t *tsdn, hpa_central_t *central, edata_t *edata, bool err = emap_split_prepare(tsdn, central->emap, &prepare, edata, size, trail, cursize - size); if (err) { - edata_cache_put(tsdn, central->edata_cache, trail); + edata_cache_small_put(tsdn, ¢ral->ecs, trail); return NULL; } emap_lock_edata2(tsdn, central->emap, edata, trail); @@ -102,7 +102,7 @@ hpa_central_alloc_grow(tsdn_t *tsdn, hpa_central_t *central, assert(edata_base_get(edata) == edata_addr_get(edata)); assert(edata_size_get(edata) >= size); assert(edata_arena_ind_get(edata) - == base_ind_get(central->edata_cache->base)); + == base_ind_get(central->ecs.fallback->base)); assert(edata_is_head_get(edata)); assert(edata_state_get(edata) == extent_state_active); assert(edata_pai_get(edata) == EXTENT_PAI_HPA); @@ -173,7 +173,7 @@ hpa_central_dalloc_merge(tsdn_t *tsdn, hpa_central_t *central, edata_t *a, edata_size_set(a, edata_size_get(a) + edata_size_get(b)); emap_merge_commit(tsdn, central->emap, &prepare, a, b); emap_unlock_edata2(tsdn, central->emap, a, b); - edata_cache_put(tsdn, central->edata_cache, b); + edata_cache_small_put(tsdn, ¢ral->ecs, b); } void diff --git a/src/pa.c b/src/pa.c index 59873c12..aee7bcd8 100644 --- a/src/pa.c +++ b/src/pa.c @@ -76,6 +76,7 @@ pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) { atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED); if (shard->ever_used_hpa) { sec_disable(tsdn, &shard->hpa_sec); + hpa_shard_disable(tsdn, &shard->hpa_shard); } } @@ -89,10 +90,10 @@ pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) { void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) { - sec_flush(tsdn, &shard->hpa_sec); pac_destroy(tsdn, &shard->pac); if (shard->ever_used_hpa) { sec_flush(tsdn, &shard->hpa_sec); + hpa_shard_disable(tsdn, &shard->hpa_shard); } }