diff --git a/include/jemalloc/internal/cache_bin.h b/include/jemalloc/internal/cache_bin.h index 1c679236..775b71f2 100644 --- a/include/jemalloc/internal/cache_bin.h +++ b/include/jemalloc/internal/cache_bin.h @@ -271,6 +271,13 @@ cache_bin_ptr_array_init_for_flush(cache_bin_ptr_array_t *arr, cache_bin_t *bin, || *arr->ptr != NULL); } +static inline void +cache_bin_ptr_array_init_for_fill(cache_bin_ptr_array_t *arr, cache_bin_t *bin, + cache_bin_sz_t nfill, szind_t ind, cache_bin_info_t *infos) { + arr->ptr = cache_bin_empty_position_get(bin, ind, infos) - nfill; + assert(cache_bin_ncached_get(bin, ind, infos) == 0); +} + JEMALLOC_ALWAYS_INLINE void * cache_bin_ptr_array_get(cache_bin_ptr_array_t *arr, cache_bin_sz_t n) { return *(arr->ptr - n); @@ -281,4 +288,17 @@ cache_bin_ptr_array_set(cache_bin_ptr_array_t *arr, cache_bin_sz_t n, void *p) { *(arr->ptr - n) = p; } +static inline void +cache_bin_fill_from_ptr_array(cache_bin_t *bin, cache_bin_ptr_array_t *arr, + szind_t ind, szind_t nfilled, cache_bin_info_t *infos) { + assert(cache_bin_ncached_get(bin, ind, infos) == 0); + if (nfilled < arr->n) { + void **empty_position = cache_bin_empty_position_get(bin, ind, + infos); + memmove(empty_position - nfilled, empty_position - arr->n, + nfilled * sizeof(void *)); + } + cache_bin_ncached_set(bin, ind, nfilled, infos); +} + #endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ diff --git a/src/arena.c b/src/arena.c index 2f8a03c0..6b5f1d31 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1327,7 +1327,9 @@ arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, const bin_info_t *bin_info = &bin_infos[binind]; const unsigned nfill = cache_bin_info_ncached_max( &tcache_bin_info[binind]) >> tcache->lg_fill_div[binind]; - void **empty_position = cache_bin_empty_position_get(tbin, binind, + + CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nfill); + cache_bin_ptr_array_init_for_fill(&ptrs, tbin, nfill, binind, tcache_bin_info); /* @@ -1374,7 +1376,7 @@ label_refill: unsigned cnt = tofill < nfree ? tofill : nfree; arena_slab_reg_alloc_batch(slabcur, bin_info, cnt, - empty_position - tofill); + &ptrs.ptr[filled]); made_progress = true; filled += cnt; continue; @@ -1403,16 +1405,9 @@ label_refill: break; } + /* OOM. */ + assert(fresh_slab == NULL); - /* - * OOM. tbin->avail isn't yet filled down to its first element, - * so the successful allocations (if any) must be moved just - * before tbin->avail before bailing out. - */ - if (filled > 0) { - memmove(empty_position - filled, empty_position - nfill, - filled * sizeof(void *)); - } assert(!alloc_and_retry); break; } /* while (filled < nfill) loop. */ @@ -1448,7 +1443,8 @@ label_refill: fresh_slab = NULL; } - cache_bin_ncached_set(tbin, binind, filled, tcache_bin_info); + cache_bin_fill_from_ptr_array(tbin, &ptrs, binind, filled, + tcache_bin_info); arena_decay_tick(tsdn, arena); }