diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h index 6783eb7f..a4bb4c0e 100644 --- a/include/jemalloc/internal/hpa.h +++ b/include/jemalloc/internal/hpa.h @@ -154,6 +154,18 @@ bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central, emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind, const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts); +edata_t *hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, + size_t alignment, bool zero, bool guarded, bool frequent_reuse, + bool *deferred_work_generated); +bool hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, + size_t old_size, size_t new_size, bool zero, + bool *deferred_work_generated); +bool hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, + size_t old_size, size_t new_size, bool *deferred_work_generated); +void hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, + bool *deferred_work_generated); +uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard); + void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src); void hpa_shard_stats_merge( tsdn_t *tsdn, hpa_shard_t *shard, hpa_shard_stats_t *dst); diff --git a/include/jemalloc/internal/pa.h b/include/jemalloc/internal/pa.h index 2c1e6369..5629b677 100644 --- a/include/jemalloc/internal/pa.h +++ b/include/jemalloc/internal/pa.h @@ -96,7 +96,7 @@ struct pa_shard_s { /* Allocates from a PAC. */ pac_t pac; - hpa_shard_t hpa_shard; + hpa_shard_t hpa; /* The source of edata_t objects. */ edata_cache_t edata_cache; diff --git a/include/jemalloc/internal/pac.h b/include/jemalloc/internal/pac.h index 069ab107..e6d87e5b 100644 --- a/include/jemalloc/internal/pac.h +++ b/include/jemalloc/internal/pac.h @@ -165,6 +165,17 @@ bool pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, ssize_t dirty_decay_ms, ssize_t muzzy_decay_ms, pac_stats_t *pac_stats, malloc_mutex_t *stats_mtx); +edata_t *pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment, + bool zero, bool guarded, bool frequent_reuse, + bool *deferred_work_generated); +bool pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size, + size_t new_size, bool zero, bool *deferred_work_generated); +bool pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size, + size_t new_size, bool *deferred_work_generated); +void pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata, + bool *deferred_work_generated); +uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac); + static inline size_t pac_mapped(const pac_t *pac) { return atomic_load_zu(&pac->stats->pac_mapped, ATOMIC_RELAXED); diff --git a/src/hpa.c b/src/hpa.c index ce6ece2b..ff64a6bb 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -8,18 +8,7 @@ #include "jemalloc/internal/witness.h" #include "jemalloc/internal/jemalloc_probe.h" -static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool guarded, bool frequent_reuse, - bool *deferred_work_generated); -static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); -static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool *deferred_work_generated); -static void hpa_dalloc( - tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated); -static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self); - -static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, +static void hpa_dalloc_batch(tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *list, bool *deferred_work_generated); const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"}; @@ -110,17 +99,6 @@ hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central, shard->stats.nhugify_failures = 0; shard->stats.ndehugifies = 0; - /* - * Fill these in last, so that if an hpa_shard gets used despite - * initialization failing, we'll at least crash instead of just - * operating on corrupted data. - */ - shard->pai.alloc = &hpa_alloc; - shard->pai.expand = &hpa_expand; - shard->pai.shrink = &hpa_shrink; - shard->pai.dalloc = &hpa_dalloc; - shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work; - err = sec_init(tsdn, &shard->sec, base, sec_opts); if (err) { return true; @@ -820,15 +798,6 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, return nsuccess; } -static hpa_shard_t * -hpa_from_pai(pai_t *self) { - assert(self->alloc == &hpa_alloc); - assert(self->expand == &hpa_expand); - assert(self->shrink == &hpa_shrink); - assert(self->dalloc == &hpa_dalloc); - return (hpa_shard_t *)self; -} - static void hpa_assert_results( tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *results) { @@ -854,9 +823,10 @@ hpa_assert_results( } } -static edata_t * -hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, - bool guarded, bool frequent_reuse, bool *deferred_work_generated) { +edata_t * +hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment, + bool zero, bool guarded, bool frequent_reuse, + bool *deferred_work_generated) { assert((size & PAGE_MASK) == 0); assert(!guarded); witness_assert_depth_to_rank( @@ -866,7 +836,6 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, if (alignment > PAGE || zero) { return NULL; } - hpa_shard_t *shard = hpa_from_pai(self); /* * frequent_use here indicates this request comes from the arena bins, @@ -907,7 +876,7 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, /* Unlikely rollback in case of overfill */ if (!edata_list_active_empty(&results)) { hpa_dalloc_batch( - tsdn, self, &results, deferred_work_generated); + tsdn, shard, &results, deferred_work_generated); } } witness_assert_depth_to_rank( @@ -915,15 +884,15 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero, return edata; } -static bool -hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, +bool +hpa_expand(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated) { /* Expand not yet supported. */ return true; } -static bool -hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, +bool +hpa_shrink(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, size_t old_size, size_t new_size, bool *deferred_work_generated) { /* Shrink not yet supported. */ return true; @@ -982,10 +951,8 @@ hpa_dalloc_locked(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata) { } static void -hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list, +hpa_dalloc_batch(tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *list, bool *deferred_work_generated) { - hpa_shard_t *shard = hpa_from_pai(self); - edata_t *edata; ql_foreach (edata, &list->head, ql_link_active) { hpa_dalloc_prepare_unlocked(tsdn, shard, edata); @@ -1003,16 +970,15 @@ hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list, malloc_mutex_unlock(tsdn, &shard->mtx); } -static void -hpa_dalloc( - tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) { +void +hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata, + bool *deferred_work_generated) { assert(!edata_guarded_get(edata)); edata_list_active_t dalloc_list; edata_list_active_init(&dalloc_list); edata_list_active_append(&dalloc_list, edata); - hpa_shard_t *shard = hpa_from_pai(self); sec_dalloc(tsdn, &shard->sec, &dalloc_list); if (edata_list_active_empty(&dalloc_list)) { /* sec consumed the pointer */ @@ -1020,17 +986,16 @@ hpa_dalloc( return; } /* We may have more than one pointer to flush now */ - hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated); + hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated); } /* * Calculate time until either purging or hugification ought to happen. * Called by background threads. */ -static uint64_t -hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { - hpa_shard_t *shard = hpa_from_pai(self); - uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX; +uint64_t +hpa_time_until_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) { + uint64_t time_ns = BACKGROUND_THREAD_DEFERRED_MAX; malloc_mutex_lock(tsdn, &shard->mtx); @@ -1090,8 +1055,7 @@ hpa_sec_flush_impl(tsdn_t *tsdn, hpa_shard_t *shard) { sec_flush(tsdn, &shard->sec, &to_flush); bool deferred_work_generated; - hpa_dalloc_batch( - tsdn, (pai_t *)shard, &to_flush, &deferred_work_generated); + hpa_dalloc_batch(tsdn, shard, &to_flush, &deferred_work_generated); } void diff --git a/src/pa.c b/src/pa.c index a03b0c1c..f14fda81 100644 --- a/src/pa.c +++ b/src/pa.c @@ -67,7 +67,7 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central, bool pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard, const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) { - if (hpa_shard_init(tsdn, &shard->hpa_shard, &shard->central->hpa, + if (hpa_shard_init(tsdn, &shard->hpa, &shard->central->hpa, shard->emap, shard->base, &shard->edata_cache, shard->ind, hpa_opts, hpa_sec_opts)) { return true; @@ -82,7 +82,7 @@ void pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) { atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED); if (shard->ever_used_hpa) { - hpa_shard_disable(tsdn, &shard->hpa_shard); + hpa_shard_disable(tsdn, &shard->hpa); } } @@ -95,7 +95,7 @@ pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) { void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) { if (shard->ever_used_hpa) { - hpa_shard_flush(tsdn, &shard->hpa_shard); + hpa_shard_flush(tsdn, &shard->hpa); } } @@ -108,16 +108,10 @@ void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) { pac_destroy(tsdn, &shard->pac); if (shard->ever_used_hpa) { - hpa_shard_destroy(tsdn, &shard->hpa_shard); + hpa_shard_destroy(tsdn, &shard->hpa); } } -static pai_t * -pa_get_pai(pa_shard_t *shard, edata_t *edata) { - return (edata_pai_get(edata) == EXTENT_PAI_PAC ? &shard->pac.pai - : &shard->hpa_shard.pai); -} - edata_t * pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, bool slab, szind_t szind, bool zero, bool guarded, @@ -128,7 +122,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, edata_t *edata = NULL; if (!guarded && pa_shard_uses_hpa(shard)) { - edata = pai_alloc(tsdn, &shard->hpa_shard.pai, size, alignment, + edata = hpa_alloc(tsdn, &shard->hpa, size, alignment, zero, /* guarded */ false, slab, deferred_work_generated); } /* @@ -136,7 +130,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment, * allocation request. */ if (edata == NULL) { - edata = pai_alloc(tsdn, &shard->pac.pai, size, alignment, zero, + edata = pac_alloc(tsdn, &shard->pac, size, alignment, zero, guarded, slab, deferred_work_generated); } if (edata != NULL) { @@ -164,10 +158,15 @@ pa_expand(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, } size_t expand_amount = new_size - old_size; - pai_t *pai = pa_get_pai(shard, edata); - - bool error = pai_expand(tsdn, pai, edata, old_size, new_size, zero, - deferred_work_generated); + /* + * HPA expand always fails (it's a stub); skip the call entirely for + * HPA-owned extents. + */ + if (edata_pai_get(edata) == EXTENT_PAI_HPA) { + return true; + } + bool error = pac_expand(tsdn, &shard->pac, edata, old_size, new_size, + zero, deferred_work_generated); if (error) { return true; } @@ -189,9 +188,15 @@ pa_shrink(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, size_t old_size, } size_t shrink_amount = old_size - new_size; - pai_t *pai = pa_get_pai(shard, edata); - bool error = pai_shrink( - tsdn, pai, edata, old_size, new_size, deferred_work_generated); + /* + * HPA shrink always fails (it's a stub); skip the call entirely for + * HPA-owned extents. + */ + if (edata_pai_get(edata) == EXTENT_PAI_HPA) { + return true; + } + bool error = pac_shrink(tsdn, &shard->pac, edata, old_size, new_size, + deferred_work_generated); if (error) { return true; } @@ -216,8 +221,11 @@ pa_dalloc(tsdn_t *tsdn, pa_shard_t *shard, edata_t *edata, edata_addr_set(edata, edata_base_get(edata)); edata_szind_set(edata, SC_NSIZES); pa_nactive_sub(shard, edata_size_get(edata) >> LG_PAGE); - pai_t *pai = pa_get_pai(shard, edata); - pai_dalloc(tsdn, pai, edata, deferred_work_generated); + if (edata_pai_get(edata) == EXTENT_PAI_HPA) { + hpa_dalloc(tsdn, &shard->hpa, edata, deferred_work_generated); + } else { + pac_dalloc(tsdn, &shard->pac, edata, deferred_work_generated); + } } bool @@ -236,14 +244,14 @@ pa_shard_set_deferral_allowed( tsdn_t *tsdn, pa_shard_t *shard, bool deferral_allowed) { if (pa_shard_uses_hpa(shard)) { hpa_shard_set_deferral_allowed( - tsdn, &shard->hpa_shard, deferral_allowed); + tsdn, &shard->hpa, deferral_allowed); } } void pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) { if (pa_shard_uses_hpa(shard)) { - hpa_shard_do_deferred_work(tsdn, &shard->hpa_shard); + hpa_shard_do_deferred_work(tsdn, &shard->hpa); } } @@ -254,14 +262,14 @@ pa_shard_do_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) { */ uint64_t pa_shard_time_until_deferred_work(tsdn_t *tsdn, pa_shard_t *shard) { - uint64_t time = pai_time_until_deferred_work(tsdn, &shard->pac.pai); + uint64_t time = pac_time_until_deferred_work(tsdn, &shard->pac); if (time == BACKGROUND_THREAD_DEFERRED_MIN) { return time; } if (pa_shard_uses_hpa(shard)) { - uint64_t hpa = pai_time_until_deferred_work( - tsdn, &shard->hpa_shard.pai); + uint64_t hpa = hpa_time_until_deferred_work( + tsdn, &shard->hpa); if (hpa < time) { time = hpa; } diff --git a/src/pa_extra.c b/src/pa_extra.c index 43dc14bf..17b4449a 100644 --- a/src/pa_extra.c +++ b/src/pa_extra.c @@ -17,7 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) { void pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) { if (shard->ever_used_hpa) { - hpa_shard_prefork2(tsdn, &shard->hpa_shard); + hpa_shard_prefork2(tsdn, &shard->hpa); } } @@ -25,7 +25,7 @@ void pa_shard_prefork3(tsdn_t *tsdn, pa_shard_t *shard) { malloc_mutex_prefork(tsdn, &shard->pac.grow_mtx); if (shard->ever_used_hpa) { - hpa_shard_prefork3(tsdn, &shard->hpa_shard); + hpa_shard_prefork3(tsdn, &shard->hpa); } } @@ -36,7 +36,7 @@ pa_shard_prefork4(tsdn_t *tsdn, pa_shard_t *shard) { ecache_prefork(tsdn, &shard->pac.ecache_retained); ecache_prefork(tsdn, &shard->pac.ecache_pinned); if (shard->ever_used_hpa) { - hpa_shard_prefork4(tsdn, &shard->hpa_shard); + hpa_shard_prefork4(tsdn, &shard->hpa); } } @@ -56,7 +56,7 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) { malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx); malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx); if (shard->ever_used_hpa) { - hpa_shard_postfork_parent(tsdn, &shard->hpa_shard); + hpa_shard_postfork_parent(tsdn, &shard->hpa); } } @@ -71,7 +71,7 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) { malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx); malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx); if (shard->ever_used_hpa) { - hpa_shard_postfork_child(tsdn, &shard->hpa_shard); + hpa_shard_postfork_child(tsdn, &shard->hpa); } } @@ -84,7 +84,7 @@ size_t pa_shard_ndirty(const pa_shard_t *shard) { size_t ndirty = ecache_npages_get(&shard->pac.ecache_dirty); if (shard->ever_used_hpa) { - ndirty += psset_ndirty(&shard->hpa_shard.psset); + ndirty += psset_ndirty(&shard->hpa.psset); } return ndirty; } @@ -177,7 +177,7 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard, } if (shard->ever_used_hpa) { - hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out); + hpa_shard_stats_merge(tsdn, &shard->hpa, hpa_stats_out); } } @@ -209,11 +209,11 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard, if (shard->ever_used_hpa) { pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, - &shard->hpa_shard.mtx, arena_prof_mutex_hpa_shard); + &shard->hpa.mtx, arena_prof_mutex_hpa_shard); pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data, - &shard->hpa_shard.grow_mtx, + &shard->hpa.grow_mtx, arena_prof_mutex_hpa_shard_grow); - sec_mutex_stats_read(tsdn, &shard->hpa_shard.sec, + sec_mutex_stats_read(tsdn, &shard->hpa.sec, &mutex_prof_data[arena_prof_mutex_hpa_sec]); } } diff --git a/src/pac.c b/src/pac.c index f589880d..aab2bb1e 100644 --- a/src/pac.c +++ b/src/pac.c @@ -4,17 +4,6 @@ #include "jemalloc/internal/pac.h" #include "jemalloc/internal/san.h" -static edata_t *pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, - size_t alignment, bool zero, bool guarded, bool frequent_reuse, - bool *deferred_work_generated); -static bool pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated); -static bool pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, - size_t old_size, size_t new_size, bool *deferred_work_generated); -static void pac_dalloc_impl( - tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated); -static uint64_t pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self); - static inline void pac_decay_data_get(pac_t *pac, extent_state_t state, decay_t **r_decay, pac_decay_stats_t **r_decay_stats, ecache_t **r_ecache) { @@ -103,12 +92,6 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap, pac->stats_mtx = stats_mtx; atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED); - pac->pai.alloc = &pac_alloc_impl; - pac->pai.expand = &pac_expand_impl; - pac->pai.shrink = &pac_shrink_impl; - pac->pai.dalloc = &pac_dalloc_impl; - pac->pai.time_until_deferred_work = &pac_time_until_deferred_work; - return false; } @@ -265,11 +248,10 @@ pac_alloc_new_guarded(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, size_t size, return edata; } -static edata_t * -pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, +edata_t * +pac_alloc(tsdn_t *tsdn, pac_t *pac, size_t size, size_t alignment, bool zero, bool guarded, bool frequent_reuse, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; ehooks_t *ehooks = pac_ehooks_get(pac); edata_t *edata = NULL; @@ -292,10 +274,11 @@ pac_alloc_impl(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, return edata; } -static bool -pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, +bool +pac_expand(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; + assert(edata_pai_get(edata) == EXTENT_PAI_PAC); + ehooks_t *ehooks = pac_ehooks_get(pac); size_t mapped_add = 0; @@ -360,10 +343,11 @@ pac_expand_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, return false; } -static bool -pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, +bool +pac_shrink(tsdn_t *tsdn, pac_t *pac, edata_t *edata, size_t old_size, size_t new_size, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; + assert(edata_pai_get(edata) == EXTENT_PAI_PAC); + ehooks_t *ehooks = pac_ehooks_get(pac); size_t shrink_amount = old_size - new_size; @@ -385,10 +369,11 @@ pac_shrink_impl(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size, return false; } -static void -pac_dalloc_impl( - tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) { - pac_t *pac = (pac_t *)self; +void +pac_dalloc(tsdn_t *tsdn, pac_t *pac, edata_t *edata, + bool *deferred_work_generated) { + assert(edata_pai_get(edata) == EXTENT_PAI_PAC); + ehooks_t *ehooks = pac_ehooks_get(pac); if (edata_guarded_get(edata)) { @@ -432,10 +417,9 @@ pac_ns_until_purge(tsdn_t *tsdn, decay_t *decay, size_t npages) { return result; } -static uint64_t -pac_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) { +uint64_t +pac_time_until_deferred_work(tsdn_t *tsdn, pac_t *pac) { uint64_t time; - pac_t *pac = (pac_t *)self; time = pac_ns_until_purge( tsdn, &pac->decay_dirty, ecache_npages_get(&pac->ecache_dirty)); diff --git a/test/stress/pa/pa_microbench.c b/test/stress/pa/pa_microbench.c index 887b955e..d21576e2 100644 --- a/test/stress/pa/pa_microbench.c +++ b/test/stress/pa/pa_microbench.c @@ -364,7 +364,7 @@ collect_hpa_stats(int shard_id, hpa_shard_stats_t *hpa_stats_out) { /* Merge HPA statistics from the shard */ hpa_shard_stats_merge( - tsdn, &g_shard_infra[shard_id].pa_shard.hpa_shard, hpa_stats_out); + tsdn, &g_shard_infra[shard_id].pa_shard.hpa, hpa_stats_out); } static void diff --git a/test/unit/hpa.c b/test/unit/hpa.c index 9c4253cd..37eb61b2 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -142,23 +142,23 @@ TEST_BEGIN(test_alloc_max) { /* Small max */ bool deferred_work_generated = false; - edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX, PAGE, false, false, + edata = hpa_alloc(tsdn, shard, ALLOC_MAX, PAGE, false, false, /* frequent_reuse */ false, &deferred_work_generated); expect_ptr_not_null(edata, "Allocation of small max failed"); - edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX + PAGE, PAGE, false, + edata = hpa_alloc(tsdn, shard, ALLOC_MAX + PAGE, PAGE, false, false, /* frequent_reuse */ false, &deferred_work_generated); expect_ptr_null(edata, "Allocation of larger than small max succeeded"); - edata = pai_alloc(tsdn, &shard->pai, ALLOC_MAX, PAGE, false, false, + edata = hpa_alloc(tsdn, shard, ALLOC_MAX, PAGE, false, false, /* frequent_reuse */ true, &deferred_work_generated); expect_ptr_not_null(edata, "Allocation of frequent reused failed"); - edata = pai_alloc(tsdn, &shard->pai, HUGEPAGE, PAGE, false, false, + edata = hpa_alloc(tsdn, shard, HUGEPAGE, PAGE, false, false, /* frequent_reuse */ true, &deferred_work_generated); expect_ptr_not_null(edata, "Allocation of frequent reused failed"); - edata = pai_alloc(tsdn, &shard->pai, HUGEPAGE + PAGE, PAGE, false, + edata = hpa_alloc(tsdn, shard, HUGEPAGE + PAGE, PAGE, false, false, /* frequent_reuse */ true, &deferred_work_generated); expect_ptr_null(edata, "Allocation of larger than hugepage succeeded"); @@ -262,7 +262,7 @@ TEST_BEGIN(test_stress) { size_t npages = npages_min + prng_range_zu( &prng_state, npages_max - npages_min); - edata_t *edata = pai_alloc(tsdn, &shard->pai, + edata_t *edata = hpa_alloc(tsdn, shard, npages * PAGE, PAGE, false, false, false, &deferred_work_generated); assert_ptr_not_null( @@ -281,7 +281,7 @@ TEST_BEGIN(test_stress) { live_edatas[victim] = live_edatas[nlive_edatas - 1]; nlive_edatas--; node_remove(&tree, to_free); - pai_dalloc(tsdn, &shard->pai, to_free, + hpa_dalloc(tsdn, shard, to_free, &deferred_work_generated); } } @@ -301,8 +301,7 @@ TEST_BEGIN(test_stress) { for (size_t i = 0; i < nlive_edatas; i++) { edata_t *to_free = live_edatas[i]; node_remove(&tree, to_free); - pai_dalloc( - tsdn, &shard->pai, to_free, &deferred_work_generated); + hpa_dalloc(tsdn, shard, to_free, &deferred_work_generated); } hpa_shard_destroy(tsdn, shard); @@ -392,7 +391,7 @@ TEST_BEGIN(test_defer_time) { tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); edata_t *edatas[HUGEPAGE_PAGES]; for (int i = 0; i < (int)HUGEPAGE_PAGES; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -408,8 +407,7 @@ TEST_BEGIN(test_defer_time) { /* Purge. Recall that dirty_mult is .25. */ for (int i = 0; i < (int)HUGEPAGE_PAGES / 2; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } hpa_shard_do_deferred_work(tsdn, shard); @@ -426,7 +424,7 @@ TEST_BEGIN(test_defer_time) { * be marked for pending hugify. */ for (int i = 0; i < (int)HUGEPAGE_PAGES / 2; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -434,7 +432,7 @@ TEST_BEGIN(test_defer_time) { * We would be ineligible for hugification, had we not already met the * threshold before dipping below it. */ - pai_dalloc(tsdn, &shard->pai, edatas[0], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[0], &deferred_work_generated); /* Wait for the threshold again. */ nstime_init2(&defer_curtime, 22, 0); hpa_shard_do_deferred_work(tsdn, shard); @@ -465,7 +463,7 @@ TEST_BEGIN(test_purge_no_infinite_loop) { const size_t size = npages * PAGE; bool deferred_work_generated = false; - edata_t *edata = pai_alloc(tsdn, &shard->pai, size, PAGE, + edata_t *edata = hpa_alloc(tsdn, shard, size, PAGE, /* zero */ false, /* guarded */ false, /* frequent_reuse */ false, &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected alloc failure"); @@ -502,10 +500,10 @@ TEST_BEGIN(test_no_min_purge_interval) { nstime_init(&defer_curtime, 0); tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); - edata_t *edata = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, + edata_t *edata = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected null edata"); - pai_dalloc(tsdn, &shard->pai, edata, &deferred_work_generated); + hpa_dalloc(tsdn, shard, edata, &deferred_work_generated); hpa_shard_do_deferred_work(tsdn, shard); /* @@ -544,10 +542,10 @@ TEST_BEGIN(test_min_purge_interval) { nstime_init(&defer_curtime, 0); tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); - edata_t *edata = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, + edata_t *edata = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected null edata"); - pai_dalloc(tsdn, &shard->pai, edata, &deferred_work_generated); + hpa_dalloc(tsdn, shard, edata, &deferred_work_generated); hpa_shard_do_deferred_work(tsdn, shard); /* @@ -597,14 +595,13 @@ TEST_BEGIN(test_purge) { enum { NALLOCS = 8 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate 3 hugepages out of 8. */ for (int i = 0; i < 3 * (int)HUGEPAGE_PAGES; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } nstime_init2(&defer_curtime, 6, 0); hpa_shard_do_deferred_work(tsdn, shard); @@ -665,14 +662,13 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { enum { NALLOCS = 8 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate 3 hugepages out of 8. */ for (int i = 0; i < 3 * (int)HUGEPAGE_PAGES; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } nstime_init2(&defer_curtime, 6, 0); hpa_shard_do_deferred_work(tsdn, shard); @@ -732,10 +728,10 @@ TEST_BEGIN(test_vectorized_opt_eq_zero) { bool deferred_work_generated = false; nstime_init(&defer_curtime, 0); tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); - edata_t *edata = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, + edata_t *edata = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected null edata"); - pai_dalloc(tsdn, &shard->pai, edata, &deferred_work_generated); + hpa_dalloc(tsdn, shard, edata, &deferred_work_generated); hpa_shard_do_deferred_work(tsdn, shard); expect_false(defer_vectorized_purge_called, "No vec purge"); @@ -775,15 +771,14 @@ TEST_BEGIN(test_starts_huge) { enum { NALLOCS = 2 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate 75% */ int pages_to_deallocate = (int)(0.75 * NALLOCS); for (int i = 0; i < pages_to_deallocate; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } /* @@ -818,7 +813,7 @@ TEST_BEGIN(test_starts_huge) { */ deferred_work_generated = false; const size_t HALF = HUGEPAGE_PAGES / 2; - edatas[1] = pai_alloc(tsdn, &shard->pai, PAGE * (HALF + 1), PAGE, false, + edatas[1] = hpa_alloc(tsdn, shard, PAGE * (HALF + 1), PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[1], "Unexpected null edata"); expect_false(deferred_work_generated, "No page is purgable"); @@ -839,7 +834,7 @@ TEST_BEGIN(test_starts_huge) { expect_zu_eq(stat->merged.nactive, HALF + (HALF + 1), "1st + 2nd"); nstime_iadd(&defer_curtime, opts.min_purge_delay_ms * 1000 * 1000); - pai_dalloc(tsdn, &shard->pai, edatas[1], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[1], &deferred_work_generated); expect_true(deferred_work_generated, ""); expect_zu_eq(stat->merged.ndirty, 3 * HALF, "1st + 2nd"); @@ -856,12 +851,11 @@ TEST_BEGIN(test_starts_huge) { /* Deallocate all the rest, but leave only two active */ for (int i = pages_to_deallocate; i < NALLOCS - 2; ++i) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } /* - * With prior pai_dalloc our last page becomes purgable, however we + * With prior hpa_dalloc our last page becomes purgable, however we * still want to respect the delay. Thus, it is not time to purge yet. */ hpa_shard_do_deferred_work(tsdn, shard); @@ -914,14 +908,13 @@ TEST_BEGIN(test_start_huge_purge_empty_only) { enum { NALLOCS = 2 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate all from the first and one PAGE from the second HP. */ for (int i = 0; i < NALLOCS / 2 + 1; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } hpa_shard_do_deferred_work(tsdn, shard); expect_true(deferred_work_generated, ""); @@ -936,9 +929,9 @@ TEST_BEGIN(test_start_huge_purge_empty_only) { expect_zu_eq(0, ndefer_purge_calls, "Should not purge anything"); /* Allocate and free 2*PAGE so that it spills into second page again */ - edatas[0] = pai_alloc(tsdn, &shard->pai, 2 * PAGE, PAGE, false, false, + edatas[0] = hpa_alloc(tsdn, shard, 2 * PAGE, PAGE, false, false, false, &deferred_work_generated); - pai_dalloc(tsdn, &shard->pai, edatas[0], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[0], &deferred_work_generated); expect_true(deferred_work_generated, ""); hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(1, ndefer_purge_calls, "Should purge, delay==0ms"); @@ -980,14 +973,13 @@ TEST_BEGIN(test_assume_huge_purge_fully) { enum { NALLOCS = HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate all */ for (int i = 0; i < NALLOCS; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } hpa_shard_do_deferred_work(tsdn, shard); expect_true(deferred_work_generated, ""); @@ -998,12 +990,12 @@ TEST_BEGIN(test_assume_huge_purge_fully) { expect_zu_eq( shard->psset.stats.empty_slabs[0].npageslabs, 1, "Non huge"); npurge_size = 0; - edatas[0] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, + edatas[0] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[0], "Unexpected null edata"); expect_zu_eq(shard->psset.stats.merged.nactive, 1, ""); expect_zu_eq(shard->psset.stats.slabs[1].npageslabs, 1, "Huge nonfull"); - pai_dalloc(tsdn, &shard->pai, edatas[0], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[0], &deferred_work_generated); expect_true(deferred_work_generated, ""); ndefer_purge_calls = 0; npurge_size = 0; @@ -1013,14 +1005,13 @@ TEST_BEGIN(test_assume_huge_purge_fully) { /* Now allocate all, free 10%, alloc 5%, assert non-huge */ for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } int ten_pct = NALLOCS / 10; for (int i = 0; i < ten_pct; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } ndefer_purge_calls = 0; npurge_size = 0; @@ -1030,7 +1021,7 @@ TEST_BEGIN(test_assume_huge_purge_fully) { ten_pct * PAGE, npurge_size, "Should purge 10 percent of pages"); for (int i = 0; i < ten_pct / 2; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -1073,21 +1064,19 @@ TEST_BEGIN(test_eager_with_purge_threshold) { enum { NALLOCS = HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate less then threshold PAGEs. */ for (size_t i = 0; i < THRESHOLD - 1; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } hpa_shard_do_deferred_work(tsdn, shard); expect_false(deferred_work_generated, "No page is purgable"); expect_zu_eq(0, ndefer_purge_calls, "Should not purge yet"); /* Deallocate one more page to meet the threshold */ - pai_dalloc( - tsdn, &shard->pai, edatas[THRESHOLD - 1], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[THRESHOLD - 1], &deferred_work_generated); hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(1, ndefer_purge_calls, "Should purge"); expect_zu_eq(shard->psset.stats.merged.ndirty, 0, ""); @@ -1126,14 +1115,13 @@ TEST_BEGIN(test_delay_when_not_allowed_deferral) { edata_t *edatas[NALLOCS]; ndefer_purge_calls = 0; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate all */ for (int i = 0; i < NALLOCS; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } /* curtime = 100.0s */ hpa_shard_do_deferred_work(tsdn, shard); @@ -1143,19 +1131,17 @@ TEST_BEGIN(test_delay_when_not_allowed_deferral) { nstime_iadd(&defer_curtime, DELAY_NS - 1); /* This activity will take the curtime=100.1 and reset purgability */ for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Dealloc all but 2 pages, purgable delay_ns later*/ for (int i = 0; i < NALLOCS - 2; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } nstime_iadd(&defer_curtime, DELAY_NS); - pai_dalloc( - tsdn, &shard->pai, edatas[NALLOCS - 1], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[NALLOCS - 1], &deferred_work_generated); expect_true(ndefer_purge_calls > 0, "Should have purged"); ndefer_purge_calls = 0; @@ -1197,22 +1183,20 @@ TEST_BEGIN(test_deferred_until_time) { edata_t *edatas[NALLOCS]; ndefer_purge_calls = 0; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } /* Deallocate 25% */ for (int i = 0; i < NALLOCS / 4; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } expect_true(deferred_work_generated, "We should hugify and purge"); /* Current time = 300ms, purge_eligible at 300ms + 1000ms */ nstime_init(&defer_curtime, 300UL * 1000 * 1000); for (int i = NALLOCS / 4; i < NALLOCS; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } expect_true(deferred_work_generated, "Purge work generated"); hpa_shard_do_deferred_work(tsdn, shard); @@ -1220,14 +1204,14 @@ TEST_BEGIN(test_deferred_until_time) { /* Current time = 900ms, purge_eligible at 1300ms */ nstime_init(&defer_curtime, 900UL * 1000 * 1000); - uint64_t until_ns = pai_time_until_deferred_work(tsdn, &shard->pai); + uint64_t until_ns = hpa_time_until_deferred_work(tsdn, shard); expect_u64_eq(until_ns, BACKGROUND_THREAD_DEFERRED_MIN, "First pass did not happen"); /* Fake that first pass happened more than min_purge_interval_ago */ nstime_init(&shard->last_purge, 350UL * 1000 * 1000); shard->stats.npurge_passes = 1; - until_ns = pai_time_until_deferred_work(tsdn, &shard->pai); + until_ns = hpa_time_until_deferred_work(tsdn, shard); expect_u64_eq(until_ns, BACKGROUND_THREAD_DEFERRED_MIN, "No need to heck anything it is more than interval"); @@ -1235,7 +1219,7 @@ TEST_BEGIN(test_deferred_until_time) { nstime_init(&defer_curtime, 1000UL * 1000 * 1000); /* Next purge expected at 900ms + min_purge_interval = 1400ms */ uint64_t expected_ms = 1400 - 1000; - until_ns = pai_time_until_deferred_work(tsdn, &shard->pai); + until_ns = hpa_time_until_deferred_work(tsdn, shard); expect_u64_eq(expected_ms, until_ns / (1000 * 1000), "Next in 400ms"); destroy_test_data(shard); } @@ -1276,7 +1260,7 @@ TEST_BEGIN(test_eager_no_hugify_on_threshold) { edata_t *edatas[NALLOCS]; ndefer_purge_calls = 0; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -1288,8 +1272,7 @@ TEST_BEGIN(test_eager_no_hugify_on_threshold) { /* Deallocate 25% */ for (int i = 0; i < NALLOCS / 4; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } expect_true(deferred_work_generated, "purge is needed"); ndefer_purge_calls = 0; @@ -1301,7 +1284,7 @@ TEST_BEGIN(test_eager_no_hugify_on_threshold) { ndefer_purge_calls = 0; nstime_iadd(&defer_curtime, 800UL * 1000 * 1000); for (int i = 0; i < NALLOCS / 4 - 1; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -1345,7 +1328,7 @@ TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall) { edata_t *edatas[NALLOCS]; ndefer_purge_calls = 0; for (int i = 0; i < NALLOCS / 2; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -1403,7 +1386,7 @@ TEST_BEGIN(test_experimental_hpa_enforce_hugify) { enum { NALLOCS = HUGEPAGE_PAGES * 95 / 100 }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -1418,8 +1401,7 @@ TEST_BEGIN(test_experimental_hpa_enforce_hugify) { /* Deallocate half to trigger purge */ for (int i = 0; i < NALLOCS / 2; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } hpa_shard_do_deferred_work(tsdn, shard); @@ -1431,7 +1413,7 @@ TEST_BEGIN(test_experimental_hpa_enforce_hugify) { "Should have triggered dehugify syscall with eager style"); for (int i = 0; i < NALLOCS / 2; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } diff --git a/test/unit/hpa_background_thread.c b/test/unit/hpa_background_thread.c index 80cf2fed..8841f5a1 100644 --- a/test/unit/hpa_background_thread.c +++ b/test/unit/hpa_background_thread.c @@ -20,7 +20,7 @@ TEST_BEGIN(test_hpa_background_thread_a0_initialized) { arena_t *a0 = arena_get(TSDN_NULL, 0, false); expect_ptr_ne(a0, NULL, ""); - bool deferral_allowed = a0->pa_shard.hpa_shard.opts.deferral_allowed; + bool deferral_allowed = a0->pa_shard.hpa.opts.deferral_allowed; expect_true(deferral_allowed, "Should have deferral_allowed option enabled for arena #0"); } diff --git a/test/unit/hpa_sec_integration.c b/test/unit/hpa_sec_integration.c index c54cdc0c..eff91579 100644 --- a/test/unit/hpa_sec_integration.c +++ b/test/unit/hpa_sec_integration.c @@ -168,7 +168,7 @@ TEST_BEGIN(test_hpa_sec) { tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); /* alloc 1 PAGE, confirm sec has fill_extra bytes. */ - edata_t *edata1 = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, + edata_t *edata1 = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edata1, "Unexpected null edata"); hpa_shard_stats_t hpa_stats; @@ -182,7 +182,7 @@ TEST_BEGIN(test_hpa_sec) { /* Alloc/dealloc NALLOCS times and confirm extents are in sec. */ edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -192,8 +192,7 @@ TEST_BEGIN(test_hpa_sec) { expect_zu_eq(hpa_stats.secstats.bytes, PAGE, "2 refills (at 0 and 4)"); for (int i = 0; i < NALLOCS - 1; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t)); hpa_shard_stats_merge(tsdn, shard, &hpa_stats); @@ -202,8 +201,7 @@ TEST_BEGIN(test_hpa_sec) { hpa_stats.secstats.bytes, sec_opts.max_bytes, "sec should be full"); /* this one should flush 1 + 0.25 * 8 = 3 extents */ - pai_dalloc( - tsdn, &shard->pai, edatas[NALLOCS - 1], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[NALLOCS - 1], &deferred_work_generated); memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t)); hpa_shard_stats_merge(tsdn, shard, &hpa_stats); expect_zu_eq(hpa_stats.psset_stats.merged.nactive, (NALLOCS - 1), ""); @@ -212,7 +210,7 @@ TEST_BEGIN(test_hpa_sec) { "sec should be full"); /* Next allocation should come from SEC and not increase active */ - edata_t *edata2 = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, + edata_t *edata2 = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edata2, "Unexpected null edata"); memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t)); @@ -222,7 +220,7 @@ TEST_BEGIN(test_hpa_sec) { "sec should have max_bytes minus one page that just came from it"); /* We return this one and it stays in the cache */ - pai_dalloc(tsdn, &shard->pai, edata2, &deferred_work_generated); + hpa_dalloc(tsdn, shard, edata2, &deferred_work_generated); memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t)); hpa_shard_stats_merge(tsdn, shard, &hpa_stats); expect_zu_eq(hpa_stats.psset_stats.merged.nactive, NALLOCS - 1, ""); diff --git a/test/unit/hpa_thp_always.c b/test/unit/hpa_thp_always.c index 6e56e663..9fbd55f9 100644 --- a/test/unit/hpa_thp_always.c +++ b/test/unit/hpa_thp_always.c @@ -177,7 +177,7 @@ TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall_thp_always) { edata_t *edatas[NALLOCS]; ndefer_purge_calls = 0; for (int i = 0; i < NALLOCS / 2; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } diff --git a/test/unit/hpa_vectorized_madvise.c b/test/unit/hpa_vectorized_madvise.c index 2121de49..e2552ec9 100644 --- a/test/unit/hpa_vectorized_madvise.c +++ b/test/unit/hpa_vectorized_madvise.c @@ -174,10 +174,10 @@ TEST_BEGIN(test_vectorized_failure_fallback) { nstime_init(&defer_curtime, 0); tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); - edata_t *edata = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false, + edata_t *edata = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edata, "Unexpected null edata"); - pai_dalloc(tsdn, &shard->pai, edata, &deferred_work_generated); + hpa_dalloc(tsdn, shard, edata, &deferred_work_generated); hpa_shard_do_deferred_work(tsdn, shard); expect_true(defer_vec_purge_didfail, "Expect vec purge fail"); @@ -218,7 +218,7 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) { enum { NALLOCS = 8 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -228,7 +228,7 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) { for (int i = 0; i < 3 * (int)HUGEPAGE_PAGES; i++) { int j = i % HUGEPAGE_PAGES; if (j != 1 && j != 3) { - pai_dalloc(tsdn, &shard->pai, edatas[i], + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } } diff --git a/test/unit/hpa_vectorized_madvise_large_batch.c b/test/unit/hpa_vectorized_madvise_large_batch.c index e92988de..10ab9b7b 100644 --- a/test/unit/hpa_vectorized_madvise_large_batch.c +++ b/test/unit/hpa_vectorized_madvise_large_batch.c @@ -170,7 +170,7 @@ TEST_BEGIN(test_vectorized_purge) { enum { NALLOCS = 8 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } @@ -180,7 +180,7 @@ TEST_BEGIN(test_vectorized_purge) { for (int i = 0; i < 3 * (int)HUGEPAGE_PAGES; i++) { int j = i % HUGEPAGE_PAGES; if (j != 1 && j != 3) { - pai_dalloc(tsdn, &shard->pai, edatas[i], + hpa_dalloc(tsdn, shard, edatas[i], &deferred_work_generated); } } @@ -232,14 +232,14 @@ TEST_BEGIN(test_purge_more_than_one_batch_pages) { enum { NALLOCS = HPA_PURGE_BATCH_MAX * 3 * HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; for (int i = 0; i < NALLOCS; i++) { - edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + edatas[i] = hpa_alloc(tsdn, shard, PAGE, PAGE, false, false, false, &deferred_work_generated); expect_ptr_not_null(edatas[i], "Unexpected null edata"); } for (int i = 0; i < HPA_PURGE_BATCH_MAX * 2 * (int)HUGEPAGE_PAGES; i++) { - pai_dalloc( - tsdn, &shard->pai, edatas[i], &deferred_work_generated); + hpa_dalloc(tsdn, shard, edatas[i], + &deferred_work_generated); } hpa_shard_do_deferred_work(tsdn, shard);