mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-06-03 02:34:17 +03:00
Remove hpa_sec_batch_fill_extra and calculate nallocs automatically.
This change includes the following improvements: - Remove the hpa_sec_batch_fill_extra parameter. - Refactor the hpa_alloc() code and helper functions to be able to allocate more than one extent out of a single pageslab. This way we can amortize the per-pageslab costs (active bitmap iteration, pageslab metadata updates) across multiple extents. - Decide on a min and max number of extents that will be allocated in hpa_alloc(). The code will try to allocate at least the min and allocate up to the max as long as we can allocate additional ones from the pageslab we already have, as additional allocations are relatively cheap. - Add extent allocation distribution stats. - Amend hpa_sec_integration.c unit test.
This commit is contained in:
Tony Printezis
Guangli Dai
parent
639e70fcfb
commit
f008ce9fe1
15 changed files with 675 additions and 105 deletions
|
|
@ -225,7 +225,8 @@ conf_error(
|
|||
/* However, tolerate experimental features. */
|
||||
return;
|
||||
}
|
||||
const char *deprecated[] = {"hpa_sec_bytes_after_flush"};
|
||||
const char *deprecated[] = {
|
||||
"hpa_sec_bytes_after_flush", "hpa_sec_batch_fill_extra"};
|
||||
const size_t deprecated_cnt = (sizeof(deprecated)
|
||||
/ sizeof(deprecated[0]));
|
||||
for (size_t i = 0; i < deprecated_cnt; ++i) {
|
||||
|
|
@ -952,9 +953,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
|
|||
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
|
||||
"hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
|
||||
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
|
||||
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
|
||||
"hpa_sec_batch_fill_extra", 1, HUGEPAGE_PAGES,
|
||||
CONF_CHECK_MIN, CONF_CHECK_MAX, true);
|
||||
|
||||
if (CONF_MATCH("slab_sizes")) {
|
||||
if (CONF_MATCH_VALUE("default")) {
|
||||
|
|
|
|||
70
src/ctl.c
70
src/ctl.c
|
|
@ -115,7 +115,6 @@ CTL_PROTO(opt_hpa_dirty_mult)
|
|||
CTL_PROTO(opt_hpa_sec_nshards)
|
||||
CTL_PROTO(opt_hpa_sec_max_alloc)
|
||||
CTL_PROTO(opt_hpa_sec_max_bytes)
|
||||
CTL_PROTO(opt_hpa_sec_batch_fill_extra)
|
||||
CTL_PROTO(opt_huge_arena_pac_thp)
|
||||
CTL_PROTO(opt_metadata_thp)
|
||||
CTL_PROTO(opt_retain)
|
||||
|
|
@ -311,6 +310,14 @@ CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_nonhuge)
|
|||
CTL_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j_ndirty_huge)
|
||||
|
||||
INDEX_PROTO(stats_arenas_i_hpa_shard_nonfull_slabs_j)
|
||||
CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_min_extents)
|
||||
CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_max_extents)
|
||||
CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_extents)
|
||||
CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_ps)
|
||||
CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_pages_per_ps)
|
||||
CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_extents_per_ps)
|
||||
CTL_PROTO(stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps)
|
||||
INDEX_PROTO(stats_arenas_i_hpa_shard_alloc_j)
|
||||
|
||||
CTL_PROTO(stats_arenas_i_nthreads)
|
||||
CTL_PROTO(stats_arenas_i_uptime)
|
||||
|
|
@ -489,7 +496,6 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
|
|||
{NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
|
||||
{NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
|
||||
{NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
|
||||
{NAME("hpa_sec_batch_fill_extra"), CTL(opt_hpa_sec_batch_fill_extra)},
|
||||
{NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
|
||||
{NAME("metadata_thp"), CTL(opt_metadata_thp)},
|
||||
{NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)},
|
||||
|
|
@ -782,6 +788,23 @@ static const ctl_named_node_t
|
|||
static const ctl_indexed_node_t stats_arenas_i_hpa_shard_nonfull_slabs_node[] =
|
||||
{{INDEX(stats_arenas_i_hpa_shard_nonfull_slabs_j)}};
|
||||
|
||||
static const ctl_named_node_t stats_arenas_i_hpa_shard_alloc_j_node[] = {
|
||||
{NAME("min_extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_min_extents)},
|
||||
{NAME("max_extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_max_extents)},
|
||||
{NAME("extents"), CTL(stats_arenas_i_hpa_shard_alloc_j_extents)},
|
||||
{NAME("ps"), CTL(stats_arenas_i_hpa_shard_alloc_j_ps)},
|
||||
{NAME("pages_per_ps"), CTL(stats_arenas_i_hpa_shard_alloc_j_pages_per_ps)},
|
||||
{NAME("extents_per_ps"),
|
||||
CTL(stats_arenas_i_hpa_shard_alloc_j_extents_per_ps)},
|
||||
{NAME("total_elapsed_ns_per_ps"),
|
||||
CTL(stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps)}};
|
||||
|
||||
static const ctl_named_node_t super_stats_arenas_i_hpa_shard_alloc_j_node[] = {
|
||||
{NAME(""), CHILD(named, stats_arenas_i_hpa_shard_alloc_j)}};
|
||||
|
||||
static const ctl_indexed_node_t stats_arenas_i_hpa_shard_alloc_node[] = {
|
||||
{INDEX(stats_arenas_i_hpa_shard_alloc_j)}};
|
||||
|
||||
static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
|
||||
{NAME("npageslabs"), CTL(stats_arenas_i_hpa_shard_npageslabs)},
|
||||
{NAME("nactive"), CTL(stats_arenas_i_hpa_shard_nactive)},
|
||||
|
|
@ -795,6 +818,8 @@ static const ctl_named_node_t stats_arenas_i_hpa_shard_node[] = {
|
|||
{NAME("nhugify_failures"), CTL(stats_arenas_i_hpa_shard_nhugify_failures)},
|
||||
{NAME("ndehugifies"), CTL(stats_arenas_i_hpa_shard_ndehugifies)},
|
||||
|
||||
{NAME("alloc"), CHILD(indexed, stats_arenas_i_hpa_shard_alloc)},
|
||||
|
||||
{NAME("full_slabs"), CHILD(named, stats_arenas_i_hpa_shard_full_slabs)},
|
||||
{NAME("empty_slabs"), CHILD(named, stats_arenas_i_hpa_shard_empty_slabs)},
|
||||
{NAME("nonfull_slabs"),
|
||||
|
|
@ -2185,8 +2210,6 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
|
|||
CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
|
||||
CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
|
||||
CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
|
||||
CTL_RO_NL_GEN(
|
||||
opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, size_t)
|
||||
CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
|
||||
CTL_RO_NL_GEN(
|
||||
opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
|
||||
|
|
@ -4100,6 +4123,36 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_nhugify_failures,
|
|||
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_ndehugifies,
|
||||
arenas_i(mib[2])->astats->hpastats.nonderived_stats.ndehugifies, uint64_t)
|
||||
|
||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_min_extents,
|
||||
arenas_i(mib[2])
|
||||
->astats->hpastats.nonderived_stats.hpa_alloc_min_extents[mib[5]],
|
||||
uint64_t);
|
||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_max_extents,
|
||||
arenas_i(mib[2])
|
||||
->astats->hpastats.nonderived_stats.hpa_alloc_max_extents[mib[5]],
|
||||
uint64_t);
|
||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_extents,
|
||||
arenas_i(mib[2])
|
||||
->astats->hpastats.nonderived_stats.hpa_alloc_extents[mib[5]],
|
||||
uint64_t);
|
||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_ps,
|
||||
arenas_i(mib[2])->astats->hpastats.nonderived_stats.hpa_alloc_ps[mib[5]],
|
||||
uint64_t);
|
||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_pages_per_ps,
|
||||
arenas_i(mib[2])
|
||||
->astats->hpastats.nonderived_stats.hpa_alloc_pages_per_ps[mib[5]],
|
||||
uint64_t);
|
||||
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_shard_alloc_j_extents_per_ps,
|
||||
arenas_i(mib[2])
|
||||
->astats->hpastats.nonderived_stats.hpa_alloc_extents_per_ps[mib[5]],
|
||||
uint64_t);
|
||||
CTL_RO_CGEN(config_stats,
|
||||
stats_arenas_i_hpa_shard_alloc_j_total_elapsed_ns_per_ps,
|
||||
arenas_i(mib[2])
|
||||
->astats->hpastats.nonderived_stats
|
||||
.hpa_alloc_total_elapsed_ns_per_ps[mib[5]],
|
||||
uint64_t);
|
||||
|
||||
/* Full, nonhuge */
|
||||
CTL_RO_CGEN(config_stats,
|
||||
stats_arenas_i_hpa_shard_full_slabs_npageslabs_nonhuge,
|
||||
|
|
@ -4193,6 +4246,15 @@ stats_arenas_i_hpa_shard_nonfull_slabs_j_index(
|
|||
return super_stats_arenas_i_hpa_shard_nonfull_slabs_j_node;
|
||||
}
|
||||
|
||||
static const ctl_named_node_t *
|
||||
stats_arenas_i_hpa_shard_alloc_j_index(
|
||||
tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t j) {
|
||||
if (j > SEC_MAX_NALLOCS) {
|
||||
return NULL;
|
||||
}
|
||||
return super_stats_arenas_i_hpa_shard_alloc_j_node;
|
||||
}
|
||||
|
||||
static bool
|
||||
ctl_arenas_i_verify(size_t i) {
|
||||
size_t a = arenas_i2a_impl(i, true, true);
|
||||
|
|
|
|||
212
src/hpa.c
212
src/hpa.c
|
|
@ -98,6 +98,19 @@ hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
|
|||
shard->stats.nhugifies = 0;
|
||||
shard->stats.nhugify_failures = 0;
|
||||
shard->stats.ndehugifies = 0;
|
||||
memset(shard->stats.hpa_alloc_min_extents, 0,
|
||||
sizeof(shard->stats.hpa_alloc_min_extents));
|
||||
memset(shard->stats.hpa_alloc_max_extents, 0,
|
||||
sizeof(shard->stats.hpa_alloc_max_extents));
|
||||
memset(shard->stats.hpa_alloc_extents, 0,
|
||||
sizeof(shard->stats.hpa_alloc_extents));
|
||||
memset(shard->stats.hpa_alloc_ps, 0, sizeof(shard->stats.hpa_alloc_ps));
|
||||
memset(shard->stats.hpa_alloc_pages_per_ps, 0,
|
||||
sizeof(shard->stats.hpa_alloc_pages_per_ps));
|
||||
memset(shard->stats.hpa_alloc_extents_per_ps, 0,
|
||||
sizeof(shard->stats.hpa_alloc_extents_per_ps));
|
||||
memset(shard->stats.hpa_alloc_total_elapsed_ns_per_ps, 0,
|
||||
sizeof(shard->stats.hpa_alloc_total_elapsed_ns_per_ps));
|
||||
|
||||
err = sec_init(tsdn, &shard->sec, base, sec_opts);
|
||||
if (err) {
|
||||
|
|
@ -123,6 +136,18 @@ hpa_shard_nonderived_stats_accum(
|
|||
dst->nhugifies += src->nhugifies;
|
||||
dst->nhugify_failures += src->nhugify_failures;
|
||||
dst->ndehugifies += src->ndehugifies;
|
||||
for (size_t i = 0; i <= SEC_MAX_NALLOCS; i++) {
|
||||
dst->hpa_alloc_min_extents[i] += src->hpa_alloc_min_extents[i];
|
||||
dst->hpa_alloc_max_extents[i] += src->hpa_alloc_max_extents[i];
|
||||
dst->hpa_alloc_extents[i] += src->hpa_alloc_extents[i];
|
||||
dst->hpa_alloc_ps[i] += src->hpa_alloc_ps[i];
|
||||
dst->hpa_alloc_pages_per_ps[i] +=
|
||||
src->hpa_alloc_pages_per_ps[i];
|
||||
dst->hpa_alloc_extents_per_ps[i] +=
|
||||
src->hpa_alloc_extents_per_ps[i];
|
||||
dst->hpa_alloc_total_elapsed_ns_per_ps[i] +=
|
||||
src->hpa_alloc_total_elapsed_ns_per_ps[i];
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
|
|
@ -629,37 +654,18 @@ hpa_shard_maybe_do_deferred_work(
|
|||
}
|
||||
|
||||
static edata_t *
|
||||
hpa_try_alloc_one_no_grow(
|
||||
tsdn_t *tsdn, hpa_shard_t *shard, size_t size, bool *oom) {
|
||||
hpa_try_alloc_one_offset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
|
||||
hpdata_t *ps, hpdata_alloc_offset_t *alloc_offset, bool *oom) {
|
||||
assert(*oom == false);
|
||||
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||
|
||||
bool err;
|
||||
edata_t *edata = edata_cache_fast_get(tsdn, &shard->ecf);
|
||||
if (edata == NULL) {
|
||||
*oom = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
|
||||
if (ps == NULL) {
|
||||
edata_cache_fast_put(tsdn, &shard->ecf, edata);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
psset_update_begin(&shard->psset, ps);
|
||||
|
||||
if (hpdata_empty(ps)) {
|
||||
/*
|
||||
* If the pageslab used to be empty, treat it as though it's
|
||||
* brand new for fragmentation-avoidance purposes; what we're
|
||||
* trying to approximate is the age of the allocations *in* that
|
||||
* pageslab, and the allocations in the new pageslab are by
|
||||
* definition the youngest in this hpa shard.
|
||||
*/
|
||||
hpdata_age_set(ps, shard->age_counter++);
|
||||
}
|
||||
|
||||
void *addr = hpdata_reserve_alloc(ps, size);
|
||||
void *addr = hpdata_reserve_alloc_offset(ps, size, alloc_offset);
|
||||
JE_USDT(hpa_alloc, 5, shard->ind, addr, size, hpdata_nactive_get(ps),
|
||||
hpdata_age_get(ps));
|
||||
edata_init(edata, shard->ind, addr, size, /* slab */ false, SC_NSIZES,
|
||||
|
|
@ -671,12 +677,12 @@ hpa_try_alloc_one_no_grow(
|
|||
/*
|
||||
* This could theoretically be moved outside of the critical section,
|
||||
* but that introduces the potential for a race. Without the lock, the
|
||||
* (initially nonempty, since this is the reuse pathway) pageslab we
|
||||
* (initially nonempty, since this is the reuse pathway) pageslab we
|
||||
* allocated out of could become otherwise empty while the lock is
|
||||
* dropped. This would force us to deal with a pageslab eviction down
|
||||
* the error pathway, which is a pain.
|
||||
*/
|
||||
err = emap_register_boundary(
|
||||
const bool err = emap_register_boundary(
|
||||
tsdn, shard->emap, edata, SC_NSIZES, /* slab */ false);
|
||||
if (err) {
|
||||
hpdata_unreserve(
|
||||
|
|
@ -693,32 +699,118 @@ hpa_try_alloc_one_no_grow(
|
|||
* principle that we didn't *really* affect shard state (we
|
||||
* tweaked the stats, but our tweaks weren't really accurate).
|
||||
*/
|
||||
psset_update_end(&shard->psset, ps);
|
||||
edata_cache_fast_put(tsdn, &shard->ecf, edata);
|
||||
*oom = true;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
|
||||
psset_update_end(&shard->psset, ps);
|
||||
return edata;
|
||||
}
|
||||
|
||||
static size_t
|
||||
hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
|
||||
size_t size, bool *oom, size_t nallocs, edata_list_active_t *results,
|
||||
hpa_try_alloc_from_one_ps(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
|
||||
size_t max_nallocs, bool *oom, edata_list_active_t *results,
|
||||
bool *deferred_work_generated) {
|
||||
assert(size <= HUGEPAGE);
|
||||
assert(size <= shard->opts.slab_max_alloc || size == sz_s2u(size));
|
||||
assert(*oom == false);
|
||||
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||
|
||||
nstime_t start;
|
||||
nstime_init_update(&start);
|
||||
|
||||
hpdata_t *ps = psset_pick_alloc(&shard->psset, size);
|
||||
if (ps == NULL) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
assert(max_nallocs <= SEC_MAX_NALLOCS);
|
||||
hpdata_alloc_offset_t alloc_offsets[SEC_MAX_NALLOCS];
|
||||
const size_t nallocs = hpdata_find_alloc_offsets(
|
||||
ps, size, alloc_offsets, max_nallocs);
|
||||
|
||||
psset_update_begin(&shard->psset, ps);
|
||||
|
||||
if (hpdata_empty(ps)) {
|
||||
/*
|
||||
* If the pageslab used to be empty, treat it as though it's
|
||||
* brand new for fragmentation-avoidance purposes; what we're
|
||||
* trying to approximate is the age of the allocations *in* that
|
||||
* pageslab, and the allocations in the new pageslab are by
|
||||
* definition the youngest in this hpa shard.
|
||||
*/
|
||||
hpdata_age_set(ps, shard->age_counter++);
|
||||
}
|
||||
|
||||
size_t nsuccess = 0;
|
||||
for (; nsuccess < nallocs; nsuccess++) {
|
||||
edata_t *edata = hpa_try_alloc_one_no_grow(
|
||||
tsdn, shard, size, oom);
|
||||
for (; nsuccess < nallocs; nsuccess += 1) {
|
||||
edata_t *edata = hpa_try_alloc_one_offset(
|
||||
tsdn, shard, size, ps, (alloc_offsets + nsuccess), oom);
|
||||
if (edata == NULL) {
|
||||
break;
|
||||
}
|
||||
|
||||
edata_list_active_append(results, edata);
|
||||
}
|
||||
|
||||
hpdata_post_reserve_alloc_offsets(ps, size, alloc_offsets, nsuccess);
|
||||
hpa_update_purge_hugify_eligibility(tsdn, shard, ps);
|
||||
psset_update_end(&shard->psset, ps);
|
||||
|
||||
const uint64_t elapsed_ns = nstime_ns_since(&start);
|
||||
assert(nsuccess <= SEC_MAX_NALLOCS);
|
||||
shard->stats.hpa_alloc_pages_per_ps[nsuccess] += nsuccess
|
||||
* (size >> LG_PAGE);
|
||||
shard->stats.hpa_alloc_extents_per_ps[nsuccess] += 1;
|
||||
shard->stats.hpa_alloc_total_elapsed_ns_per_ps[nsuccess] += elapsed_ns;
|
||||
|
||||
return nsuccess;
|
||||
}
|
||||
|
||||
static size_t
|
||||
hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
|
||||
size_t size, size_t min_nallocs, size_t max_nallocs,
|
||||
bool update_min_max_stats, bool *oom, edata_list_active_t *results,
|
||||
bool *deferred_work_generated) {
|
||||
assert(*oom == false);
|
||||
malloc_mutex_assert_owner(tsdn, &shard->mtx);
|
||||
|
||||
/*
|
||||
* As we require the shard mtx lock to update the stats,
|
||||
* we do the update the first time this function is called from
|
||||
* hpa_alloc_batch_psset().
|
||||
*/
|
||||
if (update_min_max_stats) {
|
||||
assert(min_nallocs <= SEC_MAX_NALLOCS);
|
||||
shard->stats.hpa_alloc_min_extents[min_nallocs] += 1;
|
||||
assert(max_nallocs <= SEC_MAX_NALLOCS);
|
||||
shard->stats.hpa_alloc_max_extents[max_nallocs] += 1;
|
||||
}
|
||||
|
||||
size_t nsuccess = 0;
|
||||
size_t ps_count = 0;
|
||||
while (true) {
|
||||
assert(1 <= min_nallocs);
|
||||
assert(nsuccess < min_nallocs);
|
||||
assert(min_nallocs <= max_nallocs);
|
||||
const size_t nallocs = hpa_try_alloc_from_one_ps(tsdn, shard,
|
||||
size, max_nallocs - nsuccess, oom, results,
|
||||
deferred_work_generated);
|
||||
if (nallocs == 0 || *oom) {
|
||||
break;
|
||||
}
|
||||
nsuccess += nallocs;
|
||||
ps_count += 1;
|
||||
if (min_nallocs <= nsuccess) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert(nsuccess <= SEC_MAX_NALLOCS);
|
||||
shard->stats.hpa_alloc_extents[nsuccess] += 1;
|
||||
assert(ps_count <= SEC_MAX_NALLOCS);
|
||||
shard->stats.hpa_alloc_ps[ps_count] += 1;
|
||||
|
||||
hpa_shard_maybe_do_deferred_work(tsdn, shard, /* forced */ false);
|
||||
*deferred_work_generated = hpa_shard_has_deferred_work(tsdn, shard);
|
||||
return nsuccess;
|
||||
|
|
@ -726,27 +818,26 @@ hpa_try_alloc_batch_no_grow_locked(tsdn_t *tsdn, hpa_shard_t *shard,
|
|||
|
||||
static size_t
|
||||
hpa_try_alloc_batch_no_grow(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
|
||||
bool *oom, size_t nallocs, edata_list_active_t *results,
|
||||
bool *deferred_work_generated) {
|
||||
size_t min_nallocs, size_t max_nallocs, bool update_min_max_stats,
|
||||
bool *oom, edata_list_active_t *results, bool *deferred_work_generated) {
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
size_t nsuccess = hpa_try_alloc_batch_no_grow_locked(
|
||||
tsdn, shard, size, oom, nallocs, results, deferred_work_generated);
|
||||
const size_t nsuccess = hpa_try_alloc_batch_no_grow_locked(tsdn, shard,
|
||||
size, min_nallocs, max_nallocs, update_min_max_stats, oom, results,
|
||||
deferred_work_generated);
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
return nsuccess;
|
||||
}
|
||||
|
||||
static size_t
|
||||
hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
|
||||
size_t nallocs, edata_list_active_t *results,
|
||||
size_t min_nallocs, size_t max_nallocs, edata_list_active_t *results,
|
||||
bool *deferred_work_generated) {
|
||||
assert(size <= HUGEPAGE);
|
||||
assert(size <= shard->opts.slab_max_alloc || size == sz_s2u(size));
|
||||
bool oom = false;
|
||||
|
||||
size_t nsuccess = hpa_try_alloc_batch_no_grow(
|
||||
tsdn, shard, size, &oom, nallocs, results, deferred_work_generated);
|
||||
|
||||
if (nsuccess == nallocs || oom) {
|
||||
size_t nsuccess = hpa_try_alloc_batch_no_grow(tsdn, shard, size,
|
||||
min_nallocs, max_nallocs, /* update_min_max_stats */ true, &oom,
|
||||
results, deferred_work_generated);
|
||||
if (min_nallocs <= nsuccess || oom) {
|
||||
return nsuccess;
|
||||
}
|
||||
|
||||
|
|
@ -755,13 +846,18 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
|
|||
* try to grow.
|
||||
*/
|
||||
malloc_mutex_lock(tsdn, &shard->grow_mtx);
|
||||
|
||||
/*
|
||||
* Check for grow races; maybe some earlier thread expanded the psset
|
||||
* in between when we dropped the main mutex and grabbed the grow mutex.
|
||||
*/
|
||||
nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size, &oom,
|
||||
nallocs - nsuccess, results, deferred_work_generated);
|
||||
if (nsuccess == nallocs || oom) {
|
||||
assert(nsuccess < min_nallocs);
|
||||
assert(min_nallocs <= max_nallocs);
|
||||
nsuccess += hpa_try_alloc_batch_no_grow(tsdn, shard, size,
|
||||
min_nallocs - nsuccess, max_nallocs - nsuccess,
|
||||
/* update_min_max_stats */ false, &oom, results,
|
||||
deferred_work_generated);
|
||||
if (min_nallocs <= nsuccess || oom) {
|
||||
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
|
||||
return nsuccess;
|
||||
}
|
||||
|
|
@ -785,14 +881,14 @@ hpa_alloc_batch_psset(tsdn_t *tsdn, hpa_shard_t *shard, size_t size,
|
|||
*/
|
||||
malloc_mutex_lock(tsdn, &shard->mtx);
|
||||
psset_insert(&shard->psset, ps);
|
||||
nsuccess += hpa_try_alloc_batch_no_grow_locked(tsdn, shard, size, &oom,
|
||||
nallocs - nsuccess, results, deferred_work_generated);
|
||||
assert(nsuccess < min_nallocs);
|
||||
assert(min_nallocs <= max_nallocs);
|
||||
nsuccess += hpa_try_alloc_batch_no_grow_locked(tsdn, shard, size,
|
||||
min_nallocs - nsuccess, max_nallocs - nsuccess,
|
||||
/* update_min_max_stats */ false, &oom, results,
|
||||
deferred_work_generated);
|
||||
malloc_mutex_unlock(tsdn, &shard->mtx);
|
||||
|
||||
/*
|
||||
* Drop grow_mtx before doing deferred work; other threads blocked on it
|
||||
* should be allowed to proceed while we're working.
|
||||
*/
|
||||
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
|
||||
|
||||
return nsuccess;
|
||||
|
|
@ -855,13 +951,13 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
|
|||
if (edata != NULL) {
|
||||
return edata;
|
||||
}
|
||||
size_t nallocs = sec_size_supported(&shard->sec, size)
|
||||
? shard->sec.opts.batch_fill_extra + 1
|
||||
: 1;
|
||||
edata_list_active_t results;
|
||||
edata_list_active_init(&results);
|
||||
size_t nsuccess = hpa_alloc_batch_psset(
|
||||
tsdn, shard, size, nallocs, &results, deferred_work_generated);
|
||||
size_t min_nallocs, max_nallocs;
|
||||
sec_calc_nallocs_for_size(
|
||||
&shard->sec, size, &min_nallocs, &max_nallocs);
|
||||
size_t nsuccess = hpa_alloc_batch_psset(tsdn, shard, size, min_nallocs,
|
||||
max_nallocs, &results, deferred_work_generated);
|
||||
hpa_assert_results(tsdn, shard, &results);
|
||||
edata = edata_list_active_first(&results);
|
||||
|
||||
|
|
|
|||
165
src/hpdata.c
165
src/hpdata.c
|
|
@ -170,6 +170,171 @@ hpdata_unreserve(hpdata_t *hpdata, void *addr, size_t sz) {
|
|||
hpdata_assert_consistent(hpdata);
|
||||
}
|
||||
|
||||
size_t
|
||||
hpdata_find_alloc_offsets(hpdata_t *hpdata, size_t sz,
|
||||
hpdata_alloc_offset_t *offsets, size_t max_nallocs) {
|
||||
hpdata_assert_consistent(hpdata);
|
||||
assert((sz & PAGE_MASK) == 0);
|
||||
assert(1 <= max_nallocs);
|
||||
const size_t npages = sz >> LG_PAGE;
|
||||
/* We should be able to find at least one allocation */
|
||||
assert(npages <= hpdata_longest_free_range_get(hpdata));
|
||||
|
||||
size_t nallocs = 0;
|
||||
size_t start = 0;
|
||||
size_t longest_len = 0;
|
||||
while (true) {
|
||||
size_t begin = 0;
|
||||
size_t len = 0;
|
||||
|
||||
const bool found = fb_urange_iter(
|
||||
hpdata->active_pages, HUGEPAGE_PAGES, start, &begin, &len);
|
||||
if (!found) {
|
||||
/* we should have found at least one */
|
||||
assert(0 < nallocs);
|
||||
break;
|
||||
}
|
||||
|
||||
/* carve up the free range, if it's large enough */
|
||||
while (npages <= len) {
|
||||
offsets->len_before = len;
|
||||
offsets->index = begin;
|
||||
offsets->longest_len = longest_len;
|
||||
offsets += 1;
|
||||
|
||||
nallocs += 1;
|
||||
if (nallocs == max_nallocs) {
|
||||
/* cause start to be == HUGEPAGE_PAGES to break out of the outer loop */
|
||||
begin = HUGEPAGE_PAGES;
|
||||
len = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
begin += npages;
|
||||
len -= npages;
|
||||
}
|
||||
|
||||
start = begin + len;
|
||||
assert(start <= HUGEPAGE_PAGES);
|
||||
if (start == HUGEPAGE_PAGES) {
|
||||
break;
|
||||
}
|
||||
longest_len = max_zu(longest_len, len);
|
||||
}
|
||||
|
||||
/* post-conditions */
|
||||
assert(1 <= nallocs);
|
||||
assert(nallocs <= max_nallocs);
|
||||
|
||||
return nallocs;
|
||||
}
|
||||
|
||||
void *
|
||||
hpdata_reserve_alloc_offset(
|
||||
hpdata_t *hpdata, size_t sz, hpdata_alloc_offset_t *offset) {
|
||||
/*
|
||||
* This is a metadata change; the hpdata should therefore either not be
|
||||
* in the psset, or should have explicitly marked itself as being
|
||||
* mid-update.
|
||||
*/
|
||||
assert(!hpdata->h_in_psset || hpdata->h_updating);
|
||||
assert(hpdata->h_alloc_allowed);
|
||||
assert((sz & PAGE_MASK) == 0);
|
||||
const size_t npages = sz >> LG_PAGE;
|
||||
const size_t index = offset->index;
|
||||
|
||||
fb_set_range(hpdata->active_pages, HUGEPAGE_PAGES, index, npages);
|
||||
hpdata->h_nactive += npages;
|
||||
|
||||
/*
|
||||
* We might be about to dirty some memory for the first time; update our
|
||||
* count if so.
|
||||
*/
|
||||
size_t new_dirty = fb_ucount(
|
||||
hpdata->touched_pages, HUGEPAGE_PAGES, index, npages);
|
||||
fb_set_range(hpdata->touched_pages, HUGEPAGE_PAGES, index, npages);
|
||||
hpdata->h_ntouched += new_dirty;
|
||||
|
||||
return (void *)((byte_t *)hpdata_addr_get(hpdata) + (index << LG_PAGE));
|
||||
}
|
||||
|
||||
void
|
||||
hpdata_post_reserve_alloc_offsets(hpdata_t *hpdata, size_t sz,
|
||||
hpdata_alloc_offset_t *offsets, size_t nallocs) {
|
||||
assert((sz & PAGE_MASK) == 0);
|
||||
const size_t npages = sz >> LG_PAGE;
|
||||
|
||||
if (nallocs == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t max_len = offsets[0].len_before;
|
||||
for (size_t i = 1; i < nallocs; i += 1) {
|
||||
max_len = max_zu(max_len, offsets[i].len_before);
|
||||
}
|
||||
|
||||
const size_t prev_longest = hpdata_longest_free_range_get(hpdata);
|
||||
assert(max_len <= prev_longest);
|
||||
if (max_len < prev_longest) {
|
||||
/* no need to update the hpdata longest range */
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* If we allocated out of a range that was the longest in the hpdata, it
|
||||
* might be the only one of that size and we'll have to adjust the
|
||||
* metadata.
|
||||
*/
|
||||
const size_t len_before = offsets[nallocs - 1].len_before;
|
||||
size_t start = offsets[nallocs - 1].index + len_before;
|
||||
size_t longest_len = max_zu(
|
||||
offsets[nallocs - 1].longest_len, len_before - npages);
|
||||
|
||||
const size_t rest = HUGEPAGE_PAGES - start;
|
||||
/*
|
||||
* Only look at the rest if we think we'll find a range longer than what
|
||||
* we already have. This also implicitly checks for rest == 0, so we don't
|
||||
* have to check before the first call to fb_urange_iter().
|
||||
*/
|
||||
if (longest_len < rest) {
|
||||
while (true) {
|
||||
size_t begin = 0;
|
||||
size_t len = 0;
|
||||
|
||||
const bool found = fb_urange_iter(hpdata->active_pages,
|
||||
HUGEPAGE_PAGES, start, &begin, &len);
|
||||
if (!found) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (longest_len < len) {
|
||||
if (len == prev_longest) {
|
||||
/* it's already set to the right value */
|
||||
assert(hpdata_longest_free_range_get(
|
||||
hpdata)
|
||||
== len);
|
||||
assert(fb_urange_longest(
|
||||
hpdata->active_pages,
|
||||
HUGEPAGE_PAGES)
|
||||
== len);
|
||||
return;
|
||||
}
|
||||
longest_len = len;
|
||||
}
|
||||
|
||||
start = begin + len;
|
||||
assert(start <= HUGEPAGE_PAGES);
|
||||
if (start == HUGEPAGE_PAGES) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert(fb_urange_longest(hpdata->active_pages, HUGEPAGE_PAGES)
|
||||
== longest_len);
|
||||
hpdata_longest_free_range_set(hpdata, longest_len);
|
||||
}
|
||||
|
||||
size_t
|
||||
hpdata_purge_begin(
|
||||
hpdata_t *hpdata, hpdata_purge_state_t *purge_state, size_t *nranges) {
|
||||
|
|
|
|||
|
|
@ -162,13 +162,13 @@ void (*JET_MUTABLE junk_free_callback)(
|
|||
void *ptr, size_t size) = &default_junk_free;
|
||||
void (*JET_MUTABLE invalid_conf_abort)(void) = &abort;
|
||||
|
||||
bool opt_utrace = false;
|
||||
bool opt_xmalloc = false;
|
||||
bool opt_experimental_infallible_new = false;
|
||||
bool opt_experimental_tcache_gc = true;
|
||||
bool opt_zero = false;
|
||||
unsigned opt_narenas = 0;
|
||||
fxp_t opt_narenas_ratio = FXP_INIT_INT(4);
|
||||
bool opt_utrace = false;
|
||||
bool opt_xmalloc = false;
|
||||
bool opt_experimental_infallible_new = false;
|
||||
bool opt_experimental_tcache_gc = true;
|
||||
bool opt_zero = false;
|
||||
unsigned opt_narenas = 0;
|
||||
fxp_t opt_narenas_ratio = FXP_INIT_INT(4);
|
||||
|
||||
unsigned ncpus;
|
||||
|
||||
|
|
@ -208,7 +208,6 @@ typedef struct {
|
|||
# define UTRACE(a, b, c)
|
||||
#endif
|
||||
|
||||
|
||||
/******************************************************************************/
|
||||
/*
|
||||
* Begin miscellaneous support functions.
|
||||
|
|
|
|||
29
src/sec.c
29
src/sec.c
|
|
@ -90,6 +90,35 @@ sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) {
|
|||
return &sec->bins[ind];
|
||||
}
|
||||
|
||||
void
|
||||
sec_calc_nallocs_for_size(
|
||||
sec_t *sec, size_t size, size_t *min_nallocs_ret, size_t *max_nallocs_ret) {
|
||||
size_t min_nallocs = 1;
|
||||
size_t max_nallocs = 1;
|
||||
|
||||
if (sec_size_supported(sec, size)) {
|
||||
/*
|
||||
* This attempts to fill up to 1/SEC_MAX_BYTES_DIV of the SEC.
|
||||
* If we go much over that, we might cause purging.
|
||||
* This is mainly an issue when max_bytes is small (256K)
|
||||
* and size is large. For larger max_bytes, we will
|
||||
* almost always end up with SEC_MAX_NALLOCS.
|
||||
*/
|
||||
size_t nallocs = sec->opts.max_bytes / size / SEC_MAX_BYTES_DIV;
|
||||
nallocs = max_zu(nallocs, SEC_MIN_NALLOCS);
|
||||
min_nallocs = SEC_MIN_NALLOCS;
|
||||
max_nallocs = min_zu(nallocs, SEC_MAX_NALLOCS);
|
||||
}
|
||||
|
||||
/* post-conditions */
|
||||
assert(1 <= min_nallocs);
|
||||
assert(min_nallocs <= max_nallocs);
|
||||
assert(max_nallocs <= SEC_MAX_NALLOCS);
|
||||
|
||||
*min_nallocs_ret = min_nallocs;
|
||||
*max_nallocs_ret = max_nallocs;
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
|
||||
malloc_mutex_assert_owner(tsdn, &bin->mtx);
|
||||
|
|
|
|||
81
src/stats.c
81
src/stats.c
|
|
@ -902,9 +902,7 @@ stats_arena_hpa_shard_counters_print(
|
|||
" / sec)\n"
|
||||
" Hugify failures: %" FMTu64 " (%" FMTu64
|
||||
" / sec)\n"
|
||||
" Dehugifies: %" FMTu64 " (%" FMTu64
|
||||
" / sec)\n"
|
||||
"\n",
|
||||
" Dehugifies: %" FMTu64 " (%" FMTu64 " / sec)\n",
|
||||
npageslabs, npageslabs_huge, npageslabs_nonhuge, nactive,
|
||||
nactive_huge, nactive_nonhuge, ndirty, ndirty_huge, ndirty_nonhuge,
|
||||
nretained_nonhuge, npurge_passes,
|
||||
|
|
@ -944,6 +942,82 @@ stats_arena_hpa_shard_counters_print(
|
|||
emitter_json_kv(
|
||||
emitter, "ndirty_huge", emitter_type_size, &ndirty_huge);
|
||||
emitter_json_object_end(emitter); /* End "slabs" */
|
||||
|
||||
/* alloc_batch stats */
|
||||
uint64_t hpa_alloc_min_extents[SEC_MAX_NALLOCS + 1];
|
||||
uint64_t hpa_alloc_max_extents[SEC_MAX_NALLOCS + 1];
|
||||
uint64_t hpa_alloc_extents[SEC_MAX_NALLOCS + 1];
|
||||
uint64_t hpa_alloc_ps[SEC_MAX_NALLOCS + 1];
|
||||
uint64_t hpa_alloc_pages_per_ps[SEC_MAX_NALLOCS + 1];
|
||||
uint64_t hpa_alloc_extents_per_ps[SEC_MAX_NALLOCS + 1];
|
||||
uint64_t hpa_alloc_total_elapsed_ns_per_ps[SEC_MAX_NALLOCS + 1];
|
||||
|
||||
size_t alloc_mib[CTL_MAX_DEPTH];
|
||||
CTL_LEAF_PREPARE(alloc_mib, 0, "stats.arenas");
|
||||
alloc_mib[2] = i;
|
||||
CTL_LEAF_PREPARE(alloc_mib, 3, "hpa_shard.alloc");
|
||||
|
||||
for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
|
||||
alloc_mib[5] = j;
|
||||
CTL_LEAF(alloc_mib, 6, "min_extents", &hpa_alloc_min_extents[j],
|
||||
uint64_t);
|
||||
CTL_LEAF(alloc_mib, 6, "max_extents", &hpa_alloc_max_extents[j],
|
||||
uint64_t);
|
||||
CTL_LEAF(
|
||||
alloc_mib, 6, "extents", &hpa_alloc_extents[j], uint64_t);
|
||||
CTL_LEAF(alloc_mib, 6, "ps", &hpa_alloc_ps[j], uint64_t);
|
||||
CTL_LEAF(alloc_mib, 6, "pages_per_ps",
|
||||
&hpa_alloc_pages_per_ps[j], uint64_t);
|
||||
CTL_LEAF(alloc_mib, 6, "extents_per_ps",
|
||||
&hpa_alloc_extents_per_ps[j], uint64_t);
|
||||
CTL_LEAF(alloc_mib, 6, "total_elapsed_ns_per_ps",
|
||||
&hpa_alloc_total_elapsed_ns_per_ps[j], uint64_t);
|
||||
}
|
||||
|
||||
emitter_table_printf(emitter, " extent allocation distribution:\n");
|
||||
emitter_table_printf(emitter,
|
||||
" %4s %20s %20s %20s %20s %20s %20s %24s %24s\n", "",
|
||||
"min_extents", "max_extents",
|
||||
"extents", "ps", "pages_per_ps", "extents_per_ps",
|
||||
"total_elapsed_ns_per_ps", "elapsed_ns_per_ps");
|
||||
for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
|
||||
const uint64_t extents_per_ps = hpa_alloc_extents_per_ps[j];
|
||||
const uint64_t total_elapsed_ns_per_ps =
|
||||
hpa_alloc_total_elapsed_ns_per_ps[j];
|
||||
const uint64_t elapsed_ns_per_ps = (extents_per_ps != 0)
|
||||
? (total_elapsed_ns_per_ps / extents_per_ps)
|
||||
: 0;
|
||||
emitter_table_printf(emitter,
|
||||
" %4zu %20" FMTu64 " %20" FMTu64 " %20" FMTu64
|
||||
" %20" FMTu64 " %20" FMTu64 " %20" FMTu64 " %24" FMTu64
|
||||
" %24" FMTu64 "\n",
|
||||
j, hpa_alloc_min_extents[j], hpa_alloc_max_extents[j],
|
||||
hpa_alloc_extents[j],
|
||||
hpa_alloc_ps[j], hpa_alloc_pages_per_ps[j], extents_per_ps,
|
||||
total_elapsed_ns_per_ps, elapsed_ns_per_ps);
|
||||
}
|
||||
emitter_table_printf(emitter, "\n");
|
||||
|
||||
emitter_json_array_kv_begin(emitter, "extent_allocation_distribution");
|
||||
for (size_t j = 0; j <= SEC_MAX_NALLOCS; j += 1) {
|
||||
emitter_json_object_begin(emitter);
|
||||
emitter_json_kv(emitter, "min_extents", emitter_type_uint64,
|
||||
&hpa_alloc_min_extents[j]);
|
||||
emitter_json_kv(emitter, "max_extents", emitter_type_uint64,
|
||||
&hpa_alloc_max_extents[j]);
|
||||
emitter_json_kv(emitter, "extents", emitter_type_uint64,
|
||||
&hpa_alloc_extents[j]);
|
||||
emitter_json_kv(
|
||||
emitter, "ps", emitter_type_uint64, &hpa_alloc_ps[j]);
|
||||
emitter_json_kv(emitter, "pages_per_ps", emitter_type_uint64,
|
||||
&hpa_alloc_pages_per_ps[j]);
|
||||
emitter_json_kv(emitter, "extents_per_ps", emitter_type_uint64,
|
||||
&hpa_alloc_extents_per_ps[j]);
|
||||
emitter_json_kv(emitter, "total_elapsed_ns_per_ps",
|
||||
emitter_type_uint64, &hpa_alloc_total_elapsed_ns_per_ps[j]);
|
||||
emitter_json_object_end(emitter);
|
||||
}
|
||||
emitter_json_array_end(emitter); /* End "alloc_batch" */
|
||||
}
|
||||
|
||||
static void
|
||||
|
|
@ -1687,7 +1761,6 @@ stats_general_print(emitter_t *emitter) {
|
|||
OPT_WRITE_SIZE_T("hpa_sec_nshards")
|
||||
OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
|
||||
OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
|
||||
OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
|
||||
OPT_WRITE_BOOL("huge_arena_pac_thp")
|
||||
OPT_WRITE_CHAR_P("metadata_thp")
|
||||
OPT_WRITE_INT64("mutex_max_spin")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue