diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index a64afbcf..43692671 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -207,7 +207,6 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats, hpa_shard_stats_t *hpastats); -void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena); edata_t *arena_extent_alloc_large( tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero); void arena_extent_dalloc_large_prep( @@ -216,13 +215,6 @@ void arena_extent_ralloc_large_shrink( tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize); void arena_extent_ralloc_large_expand( tsdn_t *tsdn, arena_t *arena, const edata_t *edata, size_t oldusize); -bool arena_decay_ms_set( - tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms); -ssize_t arena_decay_ms_get(arena_t *arena, extent_state_t state); -void arena_decay( - tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all); -uint64_t arena_time_until_deferred(tsdn_t *tsdn, arena_t *arena); -void arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena); void arena_reset(tsd_t *tsd, arena_t *arena); void arena_destroy(tsd_t *tsd, arena_t *arena); cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena, diff --git a/include/jemalloc/internal/arena_inlines.h b/include/jemalloc/internal/arena_inlines.h index 0e181401..f09c6491 100644 --- a/include/jemalloc/internal/arena_inlines.h +++ b/include/jemalloc/internal/arena_inlines.h @@ -308,7 +308,7 @@ arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { uint64_t *prng_state = tsd_prng_statep_get(tsd); if (unlikely(ticker_geom_ticks(decay_ticker, prng_state, nticks, tsd_reentrancy_level_get(tsd) > 0))) { - arena_decay(tsdn, arena, false, false); + pa_shard_do_deferred_work(tsdn, &arena->pa_shard, false); } } diff --git a/src/arena.c b/src/arena.c index 273bfe22..f2678bfb 100644 --- a/src/arena.c +++ b/src/arena.c @@ -91,8 +91,8 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, size_t *nactive, size_t *ndirty, size_t *nmuzzy) { *nthreads += arena_nthreads_get(arena, false); *dss = dss_prec_names[arena_dss_prec_get(arena)]; - *dirty_decay_ms = arena_decay_ms_get(arena, extent_state_dirty); - *muzzy_decay_ms = arena_decay_ms_get(arena, extent_state_muzzy); + *dirty_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_dirty); + *muzzy_decay_ms = pa_decay_ms_get(&arena->pa_shard, extent_state_muzzy); pa_shard_basic_stats_merge(&arena->pa_shard, nactive, ndirty, nmuzzy); } @@ -311,15 +311,6 @@ arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx); } -/* - * React to deferred work generated by a PAI function. Delegates to pa, which - * owns the PAC/HPA decay + background-thread interaction. - */ -void -arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) { - pa_shard_handle_deferred_work(tsdn, &arena->pa_shard); -} - static void arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); @@ -452,54 +443,12 @@ arena_extent_ralloc_large_expand( } } -bool -arena_decay_ms_set( - tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms) { - return pa_decay_ms_set(tsdn, &arena->pa_shard, state, decay_ms); -} - -ssize_t -arena_decay_ms_get(arena_t *arena, extent_state_t state) { - return pa_decay_ms_get(&arena->pa_shard, state); -} - -void -arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) { - if (all) { - /* - * A purge of "all" means "save as much memory as possible" - * (thread death, manual purge): flush caches and fully decay. - */ - pa_shard_flush(tsdn, &arena->pa_shard, /* all */ true); - return; - } - - /* - * Deferred (non-forced) decay-purge: pa owns the whole bridge (eagerness - * decision, PAC decay orchestration, and background-thread notify). - */ - pa_shard_do_deferred_work( - tsdn, &arena->pa_shard, is_background_thread); -} - -/* Called from background threads. */ -void -arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) { - /* - * Runs on the background thread. is_background_thread=true makes pa - * force the PAC purge (eagerness ALWAYS) and skip the early-wake -- - * the background thread must not notify itself. - */ - pa_shard_do_deferred_work( - tsdn, &arena->pa_shard, /* is_background_thread */ true); -} - static void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab) { bool deferred_work_generated = false; pa_dalloc(tsdn, &arena->pa_shard, slab, &deferred_work_generated); if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); + pa_shard_handle_deferred_work(tsdn, &arena->pa_shard); } } @@ -776,7 +725,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, /* zero */ false, guarded, &deferred_work_generated); if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); + pa_shard_handle_deferred_work(tsdn, &arena->pa_shard); } if (slab == NULL) { @@ -1670,13 +1619,13 @@ arena_create_huge_arena(tsd_t *tsd, unsigned ind) { */ if (!background_thread_enabled() && arena_dirty_decay_ms_default_get() > 0) { - arena_decay_ms_set( - tsd_tsdn(tsd), huge_arena, extent_state_dirty, 0); + pa_decay_ms_set( + tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_dirty, 0); } if (!background_thread_enabled() && arena_muzzy_decay_ms_default_get() > 0) { - arena_decay_ms_set( - tsd_tsdn(tsd), huge_arena, extent_state_muzzy, 0); + pa_decay_ms_set( + tsd_tsdn(tsd), &huge_arena->pa_shard, extent_state_muzzy, 0); } return huge_arena; diff --git a/src/arenas_management.c b/src/arenas_management.c index 394303a3..0eb721c8 100644 --- a/src/arenas_management.c +++ b/src/arenas_management.c @@ -202,8 +202,8 @@ arena_migrate(tsd_t *tsd, arena_t *oldarena, arena_t *newarena) { * Purge if the old arena has no associated threads anymore and * no background threads. */ - arena_decay(tsd_tsdn(tsd), oldarena, - /* is_background_thread */ false, /* all */ true); + pa_shard_flush(tsd_tsdn(tsd), &oldarena->pa_shard, + /* all */ true); } } diff --git a/src/background_thread.c b/src/background_thread.c index fe255953..4a7e65cb 100644 --- a/src/background_thread.c +++ b/src/background_thread.c @@ -344,7 +344,8 @@ background_work_sleep_once( * work that caused this thread to wake up is scheduled for. */ if (!slept_indefinitely) { - arena_do_deferred_work(tsdn, arena); + pa_shard_do_deferred_work(tsdn, &arena->pa_shard, + /* is_background_thread */ true); } if (ns_until_deferred <= BACKGROUND_THREAD_MIN_INTERVAL_NS) { /* Min interval will be used. */ diff --git a/src/ctl.c b/src/ctl.c index 021d6834..b8d7e95a 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -2616,7 +2616,7 @@ thread_idle_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, if (opt_narenas > ncpus * 2) { arena_t *arena = arena_choose(tsd, NULL); if (arena != NULL) { - arena_decay(tsd_tsdn(tsd), arena, false, true); + pa_shard_flush(tsd_tsdn(tsd), &arena->pa_shard, true); } /* * The missing arena case is not actually an error; a thread @@ -2738,7 +2738,13 @@ arena_i_decay(tsdn_t *tsdn, unsigned arena_ind, bool all) { for (unsigned i = 0; i < count; i++) { if (tarenas[i] != NULL) { - arena_decay(tsdn, tarenas[i], false, all); + if (all) { + pa_shard_flush(tsdn, &tarenas[i]->pa_shard, + true); + } else { + pa_shard_do_deferred_work( + tsdn, &tarenas[i]->pa_shard, false); + } } } } @@ -2835,7 +2841,7 @@ arena_i_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, background_thread_arena_reset_begin(tsd, arena_ind); /* Merge stats after resetting and purging arena. */ arena_reset(tsd, arena); - arena_decay(tsd_tsdn(tsd), arena, false, true); + pa_shard_flush(tsd_tsdn(tsd), &arena->pa_shard, true); ctl_darena = arenas_i(MALLCTL_ARENAS_DESTROYED); ctl_darena->initialized = true; ctl_arena_refresh(tsd_tsdn(tsd), arena, ctl_darena, arena_ind, true); @@ -2966,7 +2972,7 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, } extent_state_t state = dirty ? extent_state_dirty : extent_state_muzzy; - ssize_t oldval = arena_decay_ms_get(arena, state); + ssize_t oldval = pa_decay_ms_get(&arena->pa_shard, state); ret = ctl_read(oldp, oldlenp, &oldval, sizeof(oldval)); if (ret != 0 || newp == NULL) { return ret; @@ -2974,7 +2980,8 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, ssize_t newval; ret = ctl_write(&newval, sizeof(newval), newp, newlen); - if (ret == 0 && arena_decay_ms_set(tsd_tsdn(tsd), arena, state, newval)) { + if (ret == 0 && pa_decay_ms_set(tsd_tsdn(tsd), &arena->pa_shard, + state, newval)) { ret = EFAULT; } return ret; diff --git a/src/large.c b/src/large.c index 63de70c2..a7b4cd57 100644 --- a/src/large.c +++ b/src/large.c @@ -78,7 +78,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, edata_t *edata, size_t usize) { return true; } if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); + pa_shard_handle_deferred_work(tsdn, &arena->pa_shard); } arena_extent_ralloc_large_shrink(tsdn, arena, edata, old_usize); @@ -101,7 +101,7 @@ large_ralloc_no_move_expand( szind, zero, &deferred_work_generated); if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); + pa_shard_handle_deferred_work(tsdn, &arena->pa_shard); } if (err) { @@ -247,7 +247,7 @@ large_dalloc_finish_impl(tsdn_t *tsdn, arena_t *arena, edata_t *edata) { bool deferred_work_generated = false; pa_dalloc(tsdn, &arena->pa_shard, edata, &deferred_work_generated); if (deferred_work_generated) { - arena_handle_deferred_work(tsdn, arena); + pa_shard_handle_deferred_work(tsdn, &arena->pa_shard); } } diff --git a/src/tcache.c b/src/tcache.c index 5f6b8239..ce0b2264 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -1227,17 +1227,18 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { * tsd). Manually trigger decay to avoid pathological cases. Also * include arena 0 because the tcache array is allocated from it. */ - arena_decay( - tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), false, false); + pa_shard_do_deferred_work(tsd_tsdn(tsd), + &arena_get(tsd_tsdn(tsd), 0, false)->pa_shard, + /* is_background_thread */ false); if (arena_nthreads_get(arena, false) == 0 && !background_thread_enabled()) { /* Force purging when no threads assigned to the arena anymore. */ - arena_decay(tsd_tsdn(tsd), arena, - /* is_background_thread */ false, /* all */ true); + pa_shard_flush(tsd_tsdn(tsd), &arena->pa_shard, + /* all */ true); } else { - arena_decay(tsd_tsdn(tsd), arena, - /* is_background_thread */ false, /* all */ false); + pa_shard_do_deferred_work(tsd_tsdn(tsd), &arena->pa_shard, + /* is_background_thread */ false); } }