From 22741cfd8cdea6af06b7b77280352d5d91cc671e Mon Sep 17 00:00:00 2001 From: guangli-dai Date: Sun, 28 Jun 2026 15:00:03 -0700 Subject: [PATCH] Refactor deferral constants into corresponding headers. Currently there are deferral-relevant constants in arena.c and background_thread.c. This commit moves them out into corresponding PAC and HPA headers. Those commonly needed by both are put into a new header deferral.h where the contract of deferral are clearly stated. --- include/jemalloc/internal/arena.h | 6 ---- include/jemalloc/internal/background_thread.h | 11 ------- include/jemalloc/internal/deferral.h | 30 +++++++++++++++++++ include/jemalloc/internal/hpa.h | 8 +++++ include/jemalloc/internal/pac.h | 10 ++++++- src/arena.c | 2 +- src/background_thread.c | 5 ++-- src/hpa.c | 9 +++--- src/pa.c | 3 +- src/pac.c | 7 +++-- test/unit/hpa.c | 5 ++-- 11 files changed, 65 insertions(+), 31 deletions(-) create mode 100644 include/jemalloc/internal/deferral.h diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 51522556..a64afbcf 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -177,12 +177,6 @@ struct arena_s { /* EXTERNS */ /******************************************************************************/ -/* - * When the amount of pages to be purged exceeds this amount, deferred purge - * should happen. - */ -#define ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD UINT64_C(1024) - extern ssize_t opt_dirty_decay_ms; extern ssize_t opt_muzzy_decay_ms; diff --git a/include/jemalloc/internal/background_thread.h b/include/jemalloc/internal/background_thread.h index db26f1b8..9947a0e9 100644 --- a/include/jemalloc/internal/background_thread.h +++ b/include/jemalloc/internal/background_thread.h @@ -13,17 +13,6 @@ #define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT #define DEFAULT_NUM_BACKGROUND_THREAD 4 -/* - * These exist only as a transitional state. Eventually, deferral should be - * part of the PAI, and each implementation can indicate wait times with more - * specificity. - */ -#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_UNINITIALIZED (-2) -#define BACKGROUND_THREAD_HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000 - -#define BACKGROUND_THREAD_DEFERRED_MIN UINT64_C(0) -#define BACKGROUND_THREAD_DEFERRED_MAX UINT64_MAX - typedef enum { background_thread_stopped, background_thread_started, diff --git a/include/jemalloc/internal/deferral.h b/include/jemalloc/internal/deferral.h new file mode 100644 index 00000000..856c1c74 --- /dev/null +++ b/include/jemalloc/internal/deferral.h @@ -0,0 +1,30 @@ +#ifndef JEMALLOC_INTERNAL_DEFERRAL_H +#define JEMALLOC_INTERNAL_DEFERRAL_H + +#include "jemalloc/internal/jemalloc_preamble.h" + +/* + * The deferred-work contract shared by the page allocators that PRODUCE + * deferred work (PAC decay-purge, HPA purge/hugify) and the drivers that + * CONSUME it (the background thread, or the application-inline fallback). + * + * There is deliberately no polymorphic interface here: with only two + * page allocators, a vtable's indirect calls are needless and inefficient. + * Dispatch is direct pac_*()/hpa_*(), so the contract is a convention + * documented here, not an enforced type. The only shared code it needs is the + * constants below; each allocator keeps its own policy definition (functions + * and constants) in its own header. + * + * The convention every page allocator follows: + * - _time_until_deferred_work(): nanoseconds until its next + * deferred work is due. DEFERRED_WORK_MIN = "due now", + * DEFERRED_WORK_MAX = "nothing pending"; anything between is a real + * ns deadline. + * - _do_deferred_work(): perform whatever work is due now. + * The pa layer reports the soonest deadline across its allocators; the + * background thread sleeps until then, or indefinitely on DEFERRED_WORK_MAX. + */ +#define DEFERRED_WORK_MIN UINT64_C(0) +#define DEFERRED_WORK_MAX UINT64_MAX + +#endif /* JEMALLOC_INTERNAL_DEFERRAL_H */ diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h index b7d685d9..456e0e8c 100644 --- a/include/jemalloc/internal/hpa.h +++ b/include/jemalloc/internal/hpa.h @@ -13,6 +13,14 @@ #include "jemalloc/internal/psset.h" #include "jemalloc/internal/sec.h" +/* + * HPA-specific deferral interval bounds (currently unused). Deferral tuning + * is per-allocator policy, so it lives in the HPA header; see deferral.h for + * the shared deferred-work contract. + */ +#define HPA_INTERVAL_MAX_UNINITIALIZED (-2) +#define HPA_INTERVAL_MAX_DEFAULT_WHEN_ENABLED 5000 + typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t; struct hpa_shard_nonderived_stats_s { /* diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h index 778b265b..2910cad1 100644 --- a/include/jemalloc/internal/pac.h +++ b/include/jemalloc/internal/pac.h @@ -13,7 +13,7 @@ #include "san_bump.h" /* - * Page allocator classic; an implementation of the PAI interface that: + * Page allocator classic (PAC), a page-level allocator that: * - Can be used for arenas with custom extent hooks. * - Can always satisfy any allocation request (including highly-fragmentary * ones). @@ -28,6 +28,14 @@ enum pac_purge_eagerness_e { }; typedef enum pac_purge_eagerness_e pac_purge_eagerness_t; +/* + * When a decay sweep would purge more than this many pages, its purge is + * treated as due (worth waking the background thread for) instead of left + * to accumulate. PAC decay policy; the HPA defers on time intervals, not + * on a page count, so this constant is PAC-only. + */ +#define PAC_DECAY_PURGE_NPAGES_THRESHOLD UINT64_C(1024) + typedef struct pac_decay_stats_s pac_decay_stats_t; struct pac_decay_stats_s { /* Total number of purge sweeps. */ diff --git a/src/arena.c b/src/arena.c index ede0bc5d..86a3f701 100644 --- a/src/arena.c +++ b/src/arena.c @@ -595,7 +595,7 @@ arena_should_decay_early(tsdn_t *tsdn, arena_t *arena, decay_t *decay, } malloc_mutex_unlock(tsdn, &decay->mtx); return info->npages_to_purge_new - > ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD; + > PAC_DECAY_PURGE_NPAGES_THRESHOLD; } /* diff --git a/src/background_thread.c b/src/background_thread.c index aad84eda..837486d5 100644 --- a/src/background_thread.c +++ b/src/background_thread.c @@ -6,6 +6,7 @@ #include "jemalloc/internal/background_thread.h" #include "jemalloc/internal/background_thread_inlines.h" #include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/deferral.h" #include "jemalloc/internal/jemalloc_internal_inlines_a.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" @@ -262,7 +263,7 @@ background_thread_pause_check(tsdn_t *tsdn, background_thread_info_t *info) { static inline void background_work_sleep_once( tsdn_t *tsdn, background_thread_info_t *info, unsigned ind) { - uint64_t ns_until_deferred = BACKGROUND_THREAD_DEFERRED_MAX; + uint64_t ns_until_deferred = DEFERRED_WORK_MAX; unsigned narenas = narenas_total_get(); bool slept_indefinitely = background_thread_indefinite_sleep(info); @@ -291,7 +292,7 @@ background_work_sleep_once( } uint64_t sleep_ns; - if (ns_until_deferred == BACKGROUND_THREAD_DEFERRED_MAX) { + if (ns_until_deferred == DEFERRED_WORK_MAX) { sleep_ns = BACKGROUND_THREAD_INDEFINITE_SLEEP; } else { sleep_ns = (ns_until_deferred diff --git a/src/hpa.c b/src/hpa.c index 29ea9a44..1440b98f 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -1,6 +1,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/background_thread.h" +#include "jemalloc/internal/deferral.h" #include "jemalloc/internal/hpa.h" #include "jemalloc/internal/hpa_utils.h" #include "jemalloc/internal/jemalloc_probe.h" @@ -1081,7 +1082,7 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, */ uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) { - uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX; + uint64_t time_ns = DEFERRED_WORK_MAX; malloc_mutex_lock(tsdn, &shard->mtx); @@ -1101,7 +1102,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) { time_ns *= 1000 * 1000; } else { malloc_mutex_unlock(tsdn, &shard->mtx); - return BACKGROUND_THREAD_DEFERRED_MIN; + return DEFERRED_WORK_MIN; } } @@ -1112,7 +1113,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) { */ if (shard->stats.npurge_passes == 0) { malloc_mutex_unlock(tsdn, &shard->mtx); - return BACKGROUND_THREAD_DEFERRED_MIN; + return DEFERRED_WORK_MIN; } uint64_t since_last_purge_ms = shard->central->hooks.ms_since( &shard->last_purge); @@ -1127,7 +1128,7 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) { time_ns = until_purge_ns; } } else { - time_ns = BACKGROUND_THREAD_DEFERRED_MIN; + time_ns = DEFERRED_WORK_MIN; } } malloc_mutex_unlock(tsdn, &shard->mtx); diff --git a/src/pa.c b/src/pa.c index ca056d7e..ddd96eaa 100644 --- a/src/pa.c +++ b/src/pa.c @@ -1,6 +1,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/background_thread.h" +#include "jemalloc/internal/deferral.h" #include "jemalloc/internal/hpa.h" #include "jemalloc/internal/pa.h" @@ -272,7 +273,7 @@ pa_shard_do_deferred_work( uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) { uint64_t time = pac_time_until_deferred_work(tsdn, &shard->pac); - if (time == BACKGROUND_THREAD_DEFERRED_MIN) { + if (time == DEFERRED_WORK_MIN) { return time; } diff --git a/src/pac.c b/src/pac.c index d54e0ceb..31e7f782 100644 --- a/src/pac.c +++ b/src/pac.c @@ -2,6 +2,7 @@ #include "jemalloc/internal/arena.h" #include "jemalloc/internal/background_thread.h" +#include "jemalloc/internal/deferral.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/pac.h" #include "jemalloc/internal/san.h" @@ -482,10 +483,10 @@ static inline uint64_t pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) { if (malloc_mutex_trylock(tsdn, &decay->mtx)) { /* Use minimal interval if decay is contended. */ - return BACKGROUND_THREAD_DEFERRED_MIN; + return DEFERRED_WORK_MIN; } uint64_t result = decay_ns_until_purge( - decay, npages, ARENA_DEFERRED_PURGE_NPAGES_THRESHOLD); + decay, npages, PAC_DECAY_PURGE_NPAGES_THRESHOLD); malloc_mutex_unlock(tsdn, &decay->mtx); return result; @@ -497,7 +498,7 @@ pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac) { time = pac_ns_until_purge( tsdn, &pac->decay_dirty, ecache_npages_get(&pac->ecache_dirty)); - if (time == BACKGROUND_THREAD_DEFERRED_MIN) { + if (time == DEFERRED_WORK_MIN) { return time; } diff --git a/test/unit/hpa.c b/test/unit/hpa.c index a7acab1f..9910db2c 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -1,5 +1,6 @@ #include "test/jemalloc_test.h" +#include "jemalloc/internal/deferral.h" #include "jemalloc/internal/hpa.h" #include "jemalloc/internal/nstime.h" @@ -1131,14 +1132,14 @@ TEST_BEGIN(test_deferred_until_time) { /* Current time = 900ms, purge_eligible at 1300ms */ nstime_init(&defer_curtime, 900UL * 1000 * 1000); uint64_t until_ns = hpa_time_until_deferred_work(tsdn, shard); - expect_u64_eq(until_ns, BACKGROUND_THREAD_DEFERRED_MIN, + expect_u64_eq(until_ns, DEFERRED_WORK_MIN, "First pass did not happen"); /* Fake that first pass happened more than min_purge_interval_ago */ nstime_init(&shard->last_purge, 350UL * 1000 * 1000); shard->stats.npurge_passes = 1; until_ns = hpa_time_until_deferred_work(tsdn, shard); - expect_u64_eq(until_ns, BACKGROUND_THREAD_DEFERRED_MIN, + expect_u64_eq(until_ns, DEFERRED_WORK_MIN, "No need to heck anything it is more than interval"); nstime_init(&shard->last_purge, 900UL * 1000 * 1000);