From fa451de17fff73cc03c31ec8cd817d62927d1ff9 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Fri, 12 Apr 2024 11:08:39 -0700 Subject: [PATCH] Fix the tcache flush sanity checking around ncached and nstashed. When there were many items stashed, it's possible that after flushing stashed, ncached is already lower than the remain, in which case the flush can simply return at that point. --- src/tcache.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/tcache.c b/src/tcache.c index ca0b1acb..f9235541 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -524,11 +524,25 @@ tcache_bin_flush_impl(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, JEMALLOC_ALWAYS_INLINE void tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, szind_t binind, unsigned rem, bool small) { + assert(rem <= cache_bin_ncached_max_get(cache_bin)); assert(!tcache_bin_disabled(binind, cache_bin, tcache->tcache_slow)); + cache_bin_sz_t orig_nstashed = cache_bin_nstashed_get_local(cache_bin); tcache_bin_flush_stashed(tsd, tcache, cache_bin, binind, small); cache_bin_sz_t ncached = cache_bin_ncached_get_local(cache_bin); - assert((cache_bin_sz_t)rem <= ncached); + assert((cache_bin_sz_t)rem <= ncached + orig_nstashed); + if ((cache_bin_sz_t)rem > ncached) { + /* + * The flush_stashed above could have done enough flushing, if + * there were many items stashed. Validate that: 1) non zero + * stashed, and 2) bin stack has available space now. + */ + assert(orig_nstashed > 0); + assert(ncached + cache_bin_nstashed_get_local(cache_bin) + < cache_bin_ncached_max_get(cache_bin)); + /* Still go through the flush logic for stats purpose only. */ + rem = ncached; + } cache_bin_sz_t nflush = ncached - (cache_bin_sz_t)rem; CACHE_BIN_PTR_ARRAY_DECLARE(ptrs, nflush); @@ -537,8 +551,7 @@ tcache_bin_flush_bottom(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin, tcache_bin_flush_impl(tsd, tcache, cache_bin, binind, &ptrs, nflush, small); - cache_bin_finish_flush(cache_bin, &ptrs, - ncached - (cache_bin_sz_t)rem); + cache_bin_finish_flush(cache_bin, &ptrs, nflush); } void