mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-08-25 00:23:33 +03:00
Move deferred-work orchestration from arena into pa/pac.
Move most arena-owned PAC deferral & background thread relevant codes out to pa/pac for decoupling purposes.
This commit is contained in:
guangli-dai
Guangli Dai
parent
e16ebfe27e
commit
01df5c96e0
5 changed files with 274 additions and 202 deletions
186
src/arena.c
186
src/arena.c
|
|
@ -83,9 +83,6 @@ const arena_config_t arena_config_default = {
|
|||
* definition.
|
||||
*/
|
||||
|
||||
static void arena_maybe_do_deferred_work(
|
||||
tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new);
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
void
|
||||
|
|
@ -314,32 +311,13 @@ arena_cache_bins_stats_merge(tsdn_t *tsdn, arena_t *arena) {
|
|||
malloc_mutex_unlock(tsdn, &arena->cache_bin_array_descriptor_ql_mtx);
|
||||
}
|
||||
|
||||
static void
|
||||
arena_background_thread_inactivity_check(
|
||||
tsdn_t *tsdn, arena_t *arena, bool is_background_thread) {
|
||||
if (!background_thread_enabled() || is_background_thread) {
|
||||
return;
|
||||
}
|
||||
background_thread_info_t *info = arena_background_thread_info_get(
|
||||
arena);
|
||||
if (background_thread_indefinite_sleep(info)) {
|
||||
arena_maybe_do_deferred_work(
|
||||
tsdn, arena, &arena->pa_shard.pac.decay_dirty, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* React to deferred work generated by a PAI function.
|
||||
* React to deferred work generated by a PAI function. Delegates to pa, which
|
||||
* owns the PAC/HPA decay + background-thread interaction.
|
||||
*/
|
||||
void
|
||||
arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena) {
|
||||
witness_assert_depth_to_rank(
|
||||
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
|
||||
|
||||
if (decay_immediately(&arena->pa_shard.pac.decay_dirty)) {
|
||||
pac_decay_all_now(tsdn, &arena->pa_shard.pac, extent_state_dirty);
|
||||
}
|
||||
arena_background_thread_inactivity_check(tsdn, arena, false);
|
||||
pa_shard_handle_deferred_work(tsdn, &arena->pa_shard);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -474,29 +452,10 @@ arena_extent_ralloc_large_expand(
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* In situations where we're not forcing a decay (i.e. because the user
|
||||
* specifically requested it), should we purge ourselves, or wait for the
|
||||
* background thread to get to it.
|
||||
*/
|
||||
static pac_purge_eagerness_t
|
||||
arena_decide_unforced_purge_eagerness(bool is_background_thread) {
|
||||
if (is_background_thread) {
|
||||
return PAC_PURGE_ALWAYS;
|
||||
} else if (!is_background_thread && background_thread_enabled()) {
|
||||
return PAC_PURGE_NEVER;
|
||||
} else {
|
||||
return PAC_PURGE_ON_EPOCH_ADVANCE;
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
arena_decay_ms_set(
|
||||
tsdn_t *tsdn, arena_t *arena, extent_state_t state, ssize_t decay_ms) {
|
||||
pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
|
||||
/* is_background_thread */ false);
|
||||
return pa_decay_ms_set(
|
||||
tsdn, &arena->pa_shard, state, decay_ms, eagerness);
|
||||
return pa_decay_ms_set(tsdn, &arena->pa_shard, state, decay_ms);
|
||||
}
|
||||
|
||||
ssize_t
|
||||
|
|
@ -508,144 +467,31 @@ void
|
|||
arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
|
||||
if (all) {
|
||||
/*
|
||||
* We should take a purge of "all" to mean "save as much memory
|
||||
* as possible", including flushing any caches (for situations
|
||||
* like thread death, or manual purge calls). This blocking
|
||||
* flush-and-fully-decay path is kept separate from the deferred
|
||||
* (all=false) path below.
|
||||
* A purge of "all" means "save as much memory as possible"
|
||||
* (thread death, manual purge): flush caches and fully decay.
|
||||
*/
|
||||
pa_shard_flush(tsdn, &arena->pa_shard);
|
||||
pac_decay_all_now(
|
||||
tsdn, &arena->pa_shard.pac, extent_state_dirty);
|
||||
if (pac_should_decay_muzzy(&arena->pa_shard.pac)) {
|
||||
pac_decay_all_now(
|
||||
tsdn, &arena->pa_shard.pac, extent_state_muzzy);
|
||||
}
|
||||
pa_shard_flush(tsdn, &arena->pa_shard, /* all */ true);
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Deferred (non-forced) decay-purge. The PAC layer owns the decay
|
||||
* orchestration (lock acquisition, dirty-before-muzzy ordering, the
|
||||
* dirty-contended-skip-muzzy and muzzy short-circuit rules). We call
|
||||
* it with the eagerness decided here and then notify the background
|
||||
* thread per decay state for any epoch that advanced.
|
||||
*
|
||||
* A concurrent background_thread enable/disable (mallctl) can race
|
||||
* this path: the enable state is read lock-free twice below (for the
|
||||
* eagerness decision, then the notify guard), so the two reads may
|
||||
* disagree. Worst case is benign and self-healing:
|
||||
* disabled->enabled: purged immediately, plus a possibly-redundant
|
||||
* wake;
|
||||
* enabled->disabled: deferred but not notified this pass -- the
|
||||
* pages stay in the decay backlog and are reclaimed on the next
|
||||
* decay tick or by the bg thread before it stops.
|
||||
* It stays safe regardless: info is allocated once and never freed;
|
||||
* the wake is gated by info->mtx + background_thread_is_started() (so
|
||||
* we never signal a not-yet-running thread); the bg-thread locks here
|
||||
* are trylocks; and the purge runs under decay->mtx, which the toggle
|
||||
* never touches.
|
||||
* Deferred (non-forced) decay-purge: pa owns the whole bridge (eagerness
|
||||
* decision, PAC decay orchestration, and background-thread notify).
|
||||
*/
|
||||
pac_purge_eagerness_t eagerness = arena_decide_unforced_purge_eagerness(
|
||||
is_background_thread);
|
||||
pac_deferred_work_result_t result;
|
||||
pac_do_deferred_work(
|
||||
tsdn, &arena->pa_shard.pac, eagerness, &result);
|
||||
|
||||
if (have_background_thread && background_thread_enabled()
|
||||
&& !is_background_thread) {
|
||||
if (result.dirty_epoch_advanced) {
|
||||
arena_maybe_do_deferred_work(tsdn, arena,
|
||||
&arena->pa_shard.pac.decay_dirty,
|
||||
result.dirty_npages_new);
|
||||
}
|
||||
if (result.muzzy_epoch_advanced) {
|
||||
arena_maybe_do_deferred_work(tsdn, arena,
|
||||
&arena->pa_shard.pac.decay_muzzy,
|
||||
result.muzzy_npages_new);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static bool
|
||||
arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay,
|
||||
background_thread_info_t *info, nstime_t *remaining_sleep,
|
||||
size_t npages_new) {
|
||||
malloc_mutex_assert_owner(tsdn, &info->mtx);
|
||||
|
||||
if (malloc_mutex_trylock(tsdn, &decay->mtx)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!decay_gradually(decay)) {
|
||||
malloc_mutex_unlock(tsdn, &decay->mtx);
|
||||
return false;
|
||||
}
|
||||
|
||||
nstime_init(remaining_sleep, background_thread_wakeup_time_get(info));
|
||||
if (nstime_compare(remaining_sleep, &decay->epoch) <= 0) {
|
||||
malloc_mutex_unlock(tsdn, &decay->mtx);
|
||||
return false;
|
||||
}
|
||||
nstime_subtract(remaining_sleep, &decay->epoch);
|
||||
if (npages_new > 0) {
|
||||
uint64_t npurge_new = decay_npages_purge_in(
|
||||
decay, remaining_sleep, npages_new);
|
||||
info->npages_to_purge_new += npurge_new;
|
||||
}
|
||||
malloc_mutex_unlock(tsdn, &decay->mtx);
|
||||
return info->npages_to_purge_new
|
||||
> PAC_DECAY_PURGE_NPAGES_THRESHOLD;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check if deferred work needs to be done sooner than planned.
|
||||
* For decay we might want to wake up earlier because of an influx of dirty
|
||||
* pages. Rather than waiting for previously estimated time, we proactively
|
||||
* purge those pages.
|
||||
* If background thread sleeps indefinitely, always wake up because some
|
||||
* deferred work has been generated.
|
||||
*/
|
||||
static void
|
||||
arena_maybe_do_deferred_work(
|
||||
tsdn_t *tsdn, arena_t *arena, decay_t *decay, size_t npages_new) {
|
||||
background_thread_info_t *info = arena_background_thread_info_get(
|
||||
arena);
|
||||
if (malloc_mutex_trylock(tsdn, &info->mtx)) {
|
||||
/*
|
||||
* Background thread may hold the mutex for a long period of
|
||||
* time. We'd like to avoid the variance on application
|
||||
* threads. So keep this non-blocking, and leave the work to a
|
||||
* future epoch.
|
||||
*/
|
||||
return;
|
||||
}
|
||||
if (!background_thread_is_started(info)) {
|
||||
goto label_done;
|
||||
}
|
||||
|
||||
nstime_t remaining_sleep;
|
||||
if (background_thread_indefinite_sleep(info)) {
|
||||
background_thread_wakeup_early(info, NULL);
|
||||
} else if (arena_should_decay_early(tsdn, arena, decay, info,
|
||||
&remaining_sleep, npages_new)) {
|
||||
info->npages_to_purge_new = 0;
|
||||
background_thread_wakeup_early(info, &remaining_sleep);
|
||||
}
|
||||
label_done:
|
||||
malloc_mutex_unlock(tsdn, &info->mtx);
|
||||
pa_shard_do_deferred_work(
|
||||
tsdn, &arena->pa_shard, is_background_thread);
|
||||
}
|
||||
|
||||
/* Called from background threads. */
|
||||
void
|
||||
arena_do_deferred_work(tsdn_t *tsdn, arena_t *arena) {
|
||||
/*
|
||||
* The background thread forces decay (PAC_PURGE_ALWAYS) and drives both
|
||||
* PAC and HPA deferred work through the symmetric pa_shard facade. No
|
||||
* PAC result is needed here to notify background thread for an early
|
||||
* wake because this function should be called in background thread.
|
||||
* Runs on the background thread. is_background_thread=true makes pa
|
||||
* force the PAC purge (eagerness ALWAYS) and skip the early-wake --
|
||||
* the background thread must not notify itself.
|
||||
*/
|
||||
pa_shard_do_deferred_work(tsdn, &arena->pa_shard, PAC_PURGE_ALWAYS);
|
||||
pa_shard_do_deferred_work(
|
||||
tsdn, &arena->pa_shard, /* is_background_thread */ true);
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue