diff --git a/include/jemalloc/internal/tcache_externs.h b/include/jemalloc/internal/tcache_externs.h index 3e4a7511..83643033 100644 --- a/include/jemalloc/internal/tcache_externs.h +++ b/include/jemalloc/internal/tcache_externs.h @@ -34,7 +34,7 @@ void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, - arena_t *oldarena, arena_t *newarena); + arena_t *arena); tcache_t *tcache_get_hard(tsd_t *tsd); tcache_t *tcache_create(tsdn_t *tsdn, arena_t *arena); void tcache_cleanup(tsd_t *tsd); diff --git a/include/jemalloc/internal/tcache_structs.h b/include/jemalloc/internal/tcache_structs.h index a2b28afd..a9b70312 100644 --- a/include/jemalloc/internal/tcache_structs.h +++ b/include/jemalloc/internal/tcache_structs.h @@ -35,6 +35,7 @@ struct tcache_s { uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */ ticker_t gc_ticker; /* Drives incremental GC. */ szind_t next_gc_bin; /* Next bin to GC. */ + arena_t *arena; /* Associated arena. */ tcache_bin_t tbins[1]; /* Dynamically sized. */ /* * The pointer stacks associated with tbins follow as a contiguous diff --git a/src/ctl.c b/src/ctl.c index 83e9e93e..831877b0 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1342,7 +1342,7 @@ thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, tcache_t *tcache = tsd_tcache_get(tsd); if (tcache != NULL) { tcache_arena_reassociate(tsd_tsdn(tsd), tcache, - oldarena, newarena); + newarena); } } } diff --git a/src/jemalloc.c b/src/jemalloc.c index 7e652802..b5379cc4 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -446,6 +446,7 @@ arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) { arena_nthreads_dec(oldarena, false); arena_nthreads_inc(newarena, false); tsd_arena_set(tsd, newarena); + tsd_iarena_set(tsd, newarena); } static void diff --git a/src/tcache.c b/src/tcache.c index dff31d19..78570663 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -99,7 +99,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, assert(binind < NBINS); assert(rem <= tbin->ncached); - arena = arena_choose(tsd, NULL); + arena = tcache->arena; assert(arena != NULL); for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { /* Lock the arena bin associated with the first object. */ @@ -175,7 +175,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, assert(binind < nhbins); assert(rem <= tbin->ncached); - arena_t *arena = arena_choose(tsd, NULL); + arena_t *arena = tcache->arena; assert(arena != NULL); unsigned nflush = tbin->ncached - rem; while (nflush > 0) { @@ -259,6 +259,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, static void tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { + tcache->arena = arena; if (config_stats) { /* Link into list of extant tcaches. */ malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); @@ -269,7 +270,8 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { } static void -tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { +tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { + arena_t *arena = tcache->arena; if (config_stats) { /* Unlink from list of extant tcaches. */ malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); @@ -291,10 +293,9 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { } void -tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *oldarena, - arena_t *newarena) { - tcache_arena_dissociate(tsdn, tcache, oldarena); - tcache_arena_associate(tsdn, tcache, newarena); +tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { + tcache_arena_dissociate(tsdn, tcache); + tcache_arena_associate(tsdn, tcache, arena); } tcache_t * @@ -360,7 +361,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) { unsigned i; arena = arena_choose(tsd, NULL); - tcache_arena_dissociate(tsd_tsdn(tsd), tcache, arena); + tcache_arena_dissociate(tsd_tsdn(tsd), tcache); for (i = 0; i < NBINS; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; diff --git a/test/unit/decay.c b/test/unit/decay.c index df910aac..eb4df9d7 100644 --- a/test/unit/decay.c +++ b/test/unit/decay.c @@ -123,18 +123,28 @@ generate_dirty(unsigned arena_ind, size_t size) { TEST_BEGIN(test_decay_ticks) { ticker_t *decay_ticker; - unsigned tick0, tick1; + unsigned tick0, tick1, arena_ind; size_t sz, large0; void *p; - decay_ticker = decay_ticker_get(tsd_fetch(), 0); - assert_ptr_not_null(decay_ticker, - "Unexpected failure getting decay ticker"); - sz = sizeof(size_t); assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large0, &sz, NULL, 0), 0, "Unexpected mallctl failure"); + int err; + /* Set up a manually managed arena for test. */ + arena_ind = do_arena_create(0); + + /* Migrate to the new arena, and get the ticker. */ + unsigned old_arena_ind; + size_t sz_arena_ind = sizeof(old_arena_ind); + err = mallctl("thread.arena", (void *)&old_arena_ind, &sz_arena_ind, + (void *)&arena_ind, sizeof(arena_ind)); + assert_d_eq(err, 0, "Unexpected mallctl() failure"); + decay_ticker = decay_ticker_get(tsd_fetch(), arena_ind); + assert_ptr_not_null(decay_ticker, + "Unexpected failure getting decay ticker"); + /* * Test the standard APIs using a large size class, since we can't * control tcache interactions for small size classes (except by @@ -263,6 +273,12 @@ TEST_BEGIN(test_decay_ticks) { tcache_sizes[0] = large0; tcache_sizes[1] = 1; + size_t tcache_max, sz_tcache_max; + sz_tcache_max = sizeof(tcache_max); + err = mallctl("arenas.tcache_max", (void *)&tcache_max, + &sz_tcache_max, NULL, 0); + assert_d_eq(err, 0, "Unexpected mallctl() failure"); + sz = sizeof(unsigned); assert_d_eq(mallctl("tcache.create", (void *)&tcache_ind, &sz, NULL, 0), 0, "Unexpected mallctl failure"); @@ -285,9 +301,17 @@ TEST_BEGIN(test_decay_ticks) { (void *)&tcache_ind, sizeof(unsigned)), 0, "Unexpected mallctl failure"); tick1 = ticker_read(decay_ticker); - assert_u32_ne(tick1, tick0, - "Expected ticker to tick during tcache flush " - "(sz=%zu)", sz); + + /* Will only tick if it's in tcache. */ + if (sz <= tcache_max) { + assert_u32_ne(tick1, tick0, + "Expected ticker to tick during tcache " + "flush (sz=%zu)", sz); + } else { + assert_u32_eq(tick1, tick0, + "Unexpected ticker tick during tcache " + "flush (sz=%zu)", sz); + } } } }