diff --git a/include/jemalloc/internal/hpdata.h b/include/jemalloc/internal/hpdata.h index 7bedaf4b..d221c577 100644 --- a/include/jemalloc/internal/hpdata.h +++ b/include/jemalloc/internal/hpdata.h @@ -123,6 +123,11 @@ TYPED_LIST(hpdata_list, hpdata_t, ql_link) typedef ph(hpdata_t) hpdata_age_heap_t; ph_proto(, hpdata_age_heap_, hpdata_age_heap_t, hpdata_t); +static inline bool +hpdata_empty(hpdata_t *hpdata) { + return hpdata_nfree_get(hpdata) == HUGEPAGE_PAGES; +} + void hpdata_init(hpdata_t *hpdata, void *addr, uint64_t age); /* * Given an hpdata which can serve an allocation request, pick and reserve an diff --git a/include/jemalloc/internal/psset.h b/include/jemalloc/internal/psset.h index 01b4e80a..c876c5cf 100644 --- a/include/jemalloc/internal/psset.h +++ b/include/jemalloc/internal/psset.h @@ -64,13 +64,8 @@ void psset_stats_accum(psset_stats_t *dst, psset_stats_t *src); void psset_insert(psset_t *psset, hpdata_t *ps); void psset_remove(psset_t *psset, hpdata_t *ps); -void psset_hugify(psset_t *psset, hpdata_t *ps); - -/* - * Tries to obtain a chunk from an existing pageslab already in the set. - * Returns true on failure. - */ -bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size); +/* Analogous to the eset_fit; pick a hpdata to serve the request. */ +hpdata_t *psset_fit(psset_t *psset, size_t size); /* * Given a newly created pageslab ps (not currently in the set), pass ownership @@ -79,6 +74,7 @@ bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size); */ void psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size); +bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size); /* * Given an extent that comes from a pageslab in this pageslab set, returns it diff --git a/src/hpa.c b/src/hpa.c index 9a190c8a..56149619 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -129,7 +129,7 @@ hpa_hugify(hpdata_t *ps) { assert(hpdata_huge_get(ps)); bool err = pages_huge(hpdata_addr_get(ps), HUGEPAGE); /* - * Eat the error; even if the hugeification failed, it's still safe to + * Eat the error; even if the hugification failed, it's still safe to * pretend it didn't (and would require extraordinary measures to * unhugify). */ @@ -233,7 +233,7 @@ hpa_handle_ps_eviction(tsdn_t *tsdn, hpa_shard_t *shard, hpdata_t *ps) { /* * We do this unconditionally, even for pages which were not originally - * hugeified; it has the same effect. + * hugified; it has the same effect. */ hpa_dehugify(ps); @@ -293,7 +293,9 @@ hpa_try_alloc_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) * Do the metadata modification while holding the lock; we'll * actually change state with the lock dropped. */ - psset_hugify(&shard->psset, ps); + psset_remove(&shard->psset, ps); + hpdata_huge_set(ps, true); + psset_insert(&shard->psset, ps); } malloc_mutex_unlock(tsdn, &shard->mtx); if (hugify) { @@ -463,8 +465,8 @@ hpa_dalloc(tsdn_t *tsdn, pai_t *self, edata_t *edata) { emap_deregister_boundary(tsdn, shard->emap, edata); malloc_mutex_lock(tsdn, &shard->mtx); /* - * Note that the shard mutex protects the edata hugeified field, too. - * Page slabs can move between pssets (and have their hugeified status + * Note that the shard mutex protects the edata hugified field, too. + * Page slabs can move between pssets (and have their hugified status * change) in racy ways. */ hpdata_t *evicted_ps = psset_dalloc(&shard->psset, edata); diff --git a/src/psset.c b/src/psset.c index 54188518..2e3558c1 100644 --- a/src/psset.c +++ b/src/psset.c @@ -76,23 +76,6 @@ psset_bin_stats_remove(psset_bin_stats_t *binstats, hpdata_t *ps) { psset_bin_stats_insert_remove(binstats, ps, /* insert */ false); } -/* - * We don't currently need an "activate" equivalent to this, since down the - * allocation pathways we don't do the optimization in which we change a slab - * without first removing it from a bin. - */ -static void -psset_bin_stats_deactivate(psset_bin_stats_t *binstats, bool huge, size_t num) { - size_t *nactive_dst = huge - ? &binstats->nactive_huge : &binstats->nactive_nonhuge; - size_t *ninactive_dst = huge - ? &binstats->ninactive_huge : &binstats->ninactive_nonhuge; - - assert(*nactive_dst >= num); - *nactive_dst -= num; - *ninactive_dst += num; -} - static void psset_hpdata_heap_remove(psset_t *psset, pszind_t pind, hpdata_t *ps) { hpdata_age_heap_remove(&psset->pageslabs[pind], ps); @@ -148,32 +131,8 @@ psset_remove(psset_t *psset, hpdata_t *ps) { } } -void -psset_hugify(psset_t *psset, hpdata_t *ps) { - assert(!hpdata_huge_get(ps)); - hpdata_assert_consistent(ps); - - size_t longest_free_range = hpdata_longest_free_range_get(ps); - psset_bin_stats_t *bin_stats; - if (longest_free_range == 0) { - bin_stats = &psset->stats.full_slabs; - } else { - pszind_t pind = sz_psz2ind(sz_psz_quantize_floor( - longest_free_range << LG_PAGE)); - assert(pind < PSSET_NPSIZES); - bin_stats = &psset->stats.nonfull_slabs[pind]; - } - psset_bin_stats_remove(bin_stats, ps); - hpdata_huge_set(ps, true); - psset_bin_stats_insert(bin_stats, ps); -} - -/* - * Similar to PAC's extent_recycle_extract. Out of all the pageslabs in the - * set, picks one that can satisfy the allocation and remove it from the set. - */ -static hpdata_t * -psset_recycle_extract(psset_t *psset, size_t size) { +hpdata_t * +psset_fit(psset_t *psset, size_t size) { pszind_t min_pind = sz_psz2ind(sz_psz_quantize_ceil(size)); pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info, (size_t)min_pind); @@ -185,22 +144,14 @@ psset_recycle_extract(psset_t *psset, size_t size) { return NULL; } - psset_hpdata_heap_remove(psset, pind, ps); - if (hpdata_age_heap_empty(&psset->pageslabs[pind])) { - bitmap_set(psset->bitmap, &psset_bitmap_info, pind); - } - hpdata_assert_consistent(ps); + return ps; } -/* - * Given a pageslab ps and an edata to allocate size bytes from, initializes the - * edata with a range in the pageslab, and puts ps back in the set. - */ -static void -psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata, - size_t size) { +void +psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) { + hpdata_assert_empty(ps); size_t npages = size / PAGE; size_t begin = hpdata_reserve_alloc(ps, npages); uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE; @@ -209,30 +160,28 @@ psset_ps_alloc_insert(psset_t *psset, hpdata_t *ps, edata_t *r_edata, /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA, EXTENT_NOT_HEAD); edata_ps_set(r_edata, ps); - /* The pageslab isn't in a bin, so no bin stats need to change. */ - - size_t longest_free_range = hpdata_longest_free_range_get(ps); - if (longest_free_range == 0) { - psset_bin_stats_insert(&psset->stats.full_slabs, ps); - } else { - psset_insert(psset, ps); - } + psset_insert(psset, ps); } bool psset_alloc_reuse(psset_t *psset, edata_t *r_edata, size_t size) { - hpdata_t *ps = psset_recycle_extract(psset, size); - if (ps == NULL) { - return true; - } - psset_ps_alloc_insert(psset, ps, r_edata, size); - return false; -} + hpdata_t *ps = psset_fit(psset, size); + if (ps == NULL) { + return true; + } + psset_remove(psset, ps); -void -psset_alloc_new(psset_t *psset, hpdata_t *ps, edata_t *r_edata, size_t size) { - hpdata_assert_empty(ps); - psset_ps_alloc_insert(psset, ps, r_edata, size); + size_t npages = size / PAGE; + size_t begin = hpdata_reserve_alloc(ps, npages); + uintptr_t addr = (uintptr_t)hpdata_addr_get(ps) + begin * PAGE; + edata_init(r_edata, edata_arena_ind_get(r_edata), (void *)addr, size, + /* slab */ false, SC_NSIZES, /* sn */ 0, extent_state_active, + /* zeroed */ false, /* committed */ true, EXTENT_PAI_HPA, + EXTENT_NOT_HEAD); + edata_ps_set(r_edata, ps); + psset_insert(psset, ps); + + return false; } hpdata_t * @@ -241,70 +190,17 @@ psset_dalloc(psset_t *psset, edata_t *edata) { assert(edata_ps_get(edata) != NULL); hpdata_t *ps = edata_ps_get(edata); - size_t ps_old_longest_free_range = hpdata_longest_free_range_get(ps); - pszind_t old_pind = SC_NPSIZES; - if (ps_old_longest_free_range != 0) { - old_pind = sz_psz2ind(sz_psz_quantize_floor( - ps_old_longest_free_range << LG_PAGE)); - } - size_t begin = ((uintptr_t)edata_base_get(edata) - (uintptr_t)hpdata_addr_get(ps)) >> LG_PAGE; size_t len = edata_size_get(edata) >> LG_PAGE; - /* The pageslab is still in the bin; adjust its stats first. */ - psset_bin_stats_t *bin_stats = (ps_old_longest_free_range == 0 - ? &psset->stats.full_slabs : &psset->stats.nonfull_slabs[old_pind]); - psset_bin_stats_deactivate(bin_stats, hpdata_huge_get(ps), len); - + psset_remove(psset, ps); hpdata_unreserve(ps, begin, len); - size_t ps_new_longest_free_range = hpdata_longest_free_range_get(ps); - - /* - * If the new free range is no longer than the previous longest one, - * then the pageslab is non-empty and doesn't need to change bins. - * We're done, and don't need to return a pageslab to evict. - */ - if (ps_new_longest_free_range <= ps_old_longest_free_range) { + if (hpdata_empty(ps)) { + return ps; + } else { + psset_insert(psset, ps); return NULL; } - /* - * If it was previously non-full, then it's in some (possibly now - * incorrect) bin already; remove it. - * - * TODO: We bailed out early above if we didn't expand the longest free - * range, which should avoid a lot of redundant remove/reinserts in the - * same bin. But it doesn't eliminate all of them; it's possible that - * we decreased the longest free range length, but only slightly, and - * not enough to change our pszind. We could check that more precisely. - * (Or, ideally, size class dequantization will happen at some point, - * and the issue becomes moot). - */ - if (ps_old_longest_free_range > 0) { - psset_hpdata_heap_remove(psset, old_pind, ps); - if (hpdata_age_heap_empty(&psset->pageslabs[old_pind])) { - bitmap_set(psset->bitmap, &psset_bitmap_info, - (size_t)old_pind); - } - } else { - /* - * Otherwise, the bin was full, and we need to adjust the full - * bin stats. - */ - psset_bin_stats_remove(&psset->stats.full_slabs, ps); - } - /* If the pageslab is empty, it gets evicted from the set. */ - if (ps_new_longest_free_range == HUGEPAGE_PAGES) { - return ps; - } - /* Otherwise, it gets reinserted. */ - pszind_t new_pind = sz_psz2ind(sz_psz_quantize_floor( - ps_new_longest_free_range << LG_PAGE)); - if (hpdata_age_heap_empty(&psset->pageslabs[new_pind])) { - bitmap_unset(psset->bitmap, &psset_bitmap_info, - (size_t)new_pind); - } - psset_hpdata_heap_insert(psset, new_pind, ps); - return NULL; }