romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-15 21:27:18 +03:00
Code Issues Projects Releases Packages Wiki Activity

[SEC] Make SEC owned by hpa_shard, simplify the code, add stats, lock per bin

Browse source
This commit is contained in:
Slobodan Predolac 2025-10-30 16:05:04 -07:00 • committed by Guangli Dai
parent c7690e92da
commit 6016d86c18
35 changed files with 1264 additions and 1257 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/arena.c
View file

@ -89,7 +89,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
hpa_shard_stats_t *hpastats, sec_stats_t *secstats) {
hpa_shard_stats_t *hpastats) {
cassert(config_stats);
arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms,
@ -159,7 +159,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
}
pa_shard_stats_merge(tsdn, &arena->pa_shard, &astats->pa_shard_stats,
estats, hpastats, secstats, &astats->resident);
estats, hpastats, &astats->resident);
LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
@ -529,7 +529,7 @@ arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
* as possible", including flushing any caches (for situations
* like thread death, or manual purge calls).
*/
sec_flush(tsdn, &arena->pa_shard.hpa_sec);
pa_shard_flush(tsdn, &arena->pa_shard);
}
if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
return;

31
src/ctl.c
View file

@ -115,7 +115,6 @@ CTL_PROTO(opt_hpa_dirty_mult)
CTL_PROTO(opt_hpa_sec_nshards)
CTL_PROTO(opt_hpa_sec_max_alloc)
CTL_PROTO(opt_hpa_sec_max_bytes)
CTL_PROTO(opt_hpa_sec_bytes_after_flush)
CTL_PROTO(opt_hpa_sec_batch_fill_extra)
CTL_PROTO(opt_huge_arena_pac_thp)
CTL_PROTO(opt_metadata_thp)
@ -339,6 +338,11 @@ CTL_PROTO(stats_arenas_i_tcache_stashed_bytes)
CTL_PROTO(stats_arenas_i_resident)
CTL_PROTO(stats_arenas_i_abandoned_vm)
CTL_PROTO(stats_arenas_i_hpa_sec_bytes)
CTL_PROTO(stats_arenas_i_hpa_sec_hits)
CTL_PROTO(stats_arenas_i_hpa_sec_misses)
CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_flush)
CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_noflush)
CTL_PROTO(stats_arenas_i_hpa_sec_overfills)
INDEX_PROTO(stats_arenas_i)
CTL_PROTO(stats_allocated)
CTL_PROTO(stats_active)
@ -486,7 +490,6 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
{NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
{NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
{NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
{NAME("hpa_sec_bytes_after_flush"), CTL(opt_hpa_sec_bytes_after_flush)},
{NAME("hpa_sec_batch_fill_extra"), CTL(opt_hpa_sec_batch_fill_extra)},
{NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
{NAME("metadata_thp"), CTL(opt_metadata_thp)},
@ -826,6 +829,12 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("resident"), CTL(stats_arenas_i_resident)},
{NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)},
{NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)},
{NAME("hpa_sec_hits"), CTL(stats_arenas_i_hpa_sec_hits)},
{NAME("hpa_sec_misses"), CTL(stats_arenas_i_hpa_sec_misses)},
{NAME("hpa_sec_dalloc_noflush"),
CTL(stats_arenas_i_hpa_sec_dalloc_noflush)},
{NAME("hpa_sec_dalloc_flush"), CTL(stats_arenas_i_hpa_sec_dalloc_flush)},
{NAME("hpa_sec_overfills"), CTL(stats_arenas_i_hpa_sec_overfills)},
{NAME("small"), CHILD(named, stats_arenas_i_small)},
{NAME("large"), CHILD(named, stats_arenas_i_large)},
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
@ -1066,7 +1075,7 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
&ctl_arena->pdirty, &ctl_arena->pmuzzy,
&ctl_arena->astats->astats, ctl_arena->astats->bstats,
ctl_arena->astats->lstats, ctl_arena->astats->estats,
&ctl_arena->astats->hpastats, &ctl_arena->astats->secstats);
&ctl_arena->astats->hpastats);
for (i = 0; i < SC_NBINS; i++) {
bin_stats_t *bstats =
@ -1258,7 +1267,6 @@ ctl_arena_stats_sdmerge(
/* Merge HPA stats. */
hpa_shard_stats_accum(&sdstats->hpastats, &astats->hpastats);
sec_stats_accum(&sdstats->secstats, &astats->secstats);
}
}
@ -2175,11 +2183,8 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
CTL_RO_NL_GEN(
opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush, size_t)
CTL_RO_NL_GEN(
opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, size_t)
CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
CTL_RO_NL_GEN(
opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
@ -3869,7 +3874,17 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm,
size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_bytes,
arenas_i(mib[2])->astats->secstats.bytes, size_t)
arenas_i(mib[2])->astats->hpastats.secstats.bytes, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_hits,
arenas_i(mib[2])->astats->hpastats.secstats.total.nhits, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_misses,
arenas_i(mib[2])->astats->hpastats.secstats.total.nmisses, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_flush,
arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_flush, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_noflush,
arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_noflush, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_overfills,
arenas_i(mib[2])->astats->hpastats.secstats.total.noverfills, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
arenas_i(mib[2])->astats->allocated_small, size_t)

144
src/hpa.c
View file

@ -11,19 +11,17 @@
static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated);
static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void hpa_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated);
static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated);
const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
bool opt_experimental_hpa_start_huge_if_thp_always = true;
@ -74,9 +72,9 @@ hpa_do_consistency_checks(hpa_shard_t *shard) {
}
bool
hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts) {
hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts) {
/* malloc_conf processing should have filtered out these cases. */
assert(hpa_supported());
bool err;
@ -118,13 +116,16 @@ hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
* operating on corrupted data.
*/
shard->pai.alloc = &hpa_alloc;
shard->pai.alloc_batch = &hpa_alloc_batch;
shard->pai.expand = &hpa_expand;
shard->pai.shrink = &hpa_shrink;
shard->pai.dalloc = &hpa_dalloc;
shard->pai.dalloc_batch = &hpa_dalloc_batch;
shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work;
err = sec_init(tsdn, &shard->sec, base, sec_opts);
if (err) {
return true;
}
hpa_do_consistency_checks(shard);
return false;
@ -151,6 +152,7 @@ hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src) {
psset_stats_accum(&dst->psset_stats, &src->psset_stats);
hpa_shard_nonderived_stats_accum(
&dst->nonderived_stats, &src->nonderived_stats);
sec_stats_accum(&dst->secstats, &src->secstats);
}
void
@ -164,6 +166,8 @@ hpa_shard_stats_merge(
hpa_shard_nonderived_stats_accum(&dst->nonderived_stats, &shard->stats);
malloc_mutex_unlock(tsdn, &shard->mtx);
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
sec_stats_merge(tsdn, &shard->sec, &dst->secstats);
}
static bool
@ -825,37 +829,9 @@ hpa_from_pai(pai_t *self) {
return (hpa_shard_t *)self;
}
static size_t
hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated) {
assert(nallocs > 0);
assert((size & PAGE_MASK) == 0);
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
hpa_shard_t *shard = hpa_from_pai(self);
/*
* frequent_use here indicates this request comes from the arena bins,
* in which case it will be split into slabs, and therefore there is no
* intrinsic slack in the allocation (the entire range of allocated size
* will be accessed).
*
* In this case bypass the slab_max_alloc limit (if still within the
* huge page size). These requests do not concern internal
* fragmentation with huge pages (again, the full size will be used).
*/
if (!(frequent_reuse && size <= HUGEPAGE)
&& (size > shard->opts.slab_max_alloc)) {
return 0;
}
size_t nsuccess = hpa_alloc_batch_psset(
tsdn, shard, size, nallocs, results, deferred_work_generated);
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
static void
hpa_assert_results(
tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *results) {
/*
* Guard the sanity checks with config_debug because the loop cannot be
* proven non-circular by the compiler, even if everything within the
@ -876,7 +852,6 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
assert(edata_base_get(edata) != NULL);
}
}
return nsuccess;
}
static edata_t *
@ -891,16 +866,52 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
if (alignment > PAGE || zero) {
return NULL;
}
hpa_shard_t *shard = hpa_from_pai(self);
/*
* An alloc with alignment == PAGE and zero == false is equivalent to a
* batch alloc of 1. Just do that, so we can share code.
* frequent_use here indicates this request comes from the arena bins,
* in which case it will be split into slabs, and therefore there is no
* intrinsic slack in the allocation (the entire range of allocated size
* will be accessed).
*
* In this case bypass the slab_max_alloc limit (if still within the
* huge page size). These requests do not concern internal
* fragmentation with huge pages (again, the full size will be used).
*/
if (!(frequent_reuse && size <= HUGEPAGE)
&& (size > shard->opts.slab_max_alloc)) {
return NULL;
}
edata_t *edata = sec_alloc(tsdn, &shard->sec, size);
if (edata != NULL) {
return edata;
}
size_t nallocs = sec_size_supported(&shard->sec, size)
? shard->sec.opts.batch_fill_extra + 1
: 1;
edata_list_active_t results;
edata_list_active_init(&results);
size_t nallocs = hpa_alloc_batch(tsdn, self, size, /* nallocs */ 1,
&results, frequent_reuse, deferred_work_generated);
assert(nallocs == 0 || nallocs == 1);
edata_t *edata = edata_list_active_first(&results);
size_t nsuccess = hpa_alloc_batch_psset(
tsdn, shard, size, nallocs, &results, deferred_work_generated);
hpa_assert_results(tsdn, shard, &results);
edata = edata_list_active_first(&results);
if (edata != NULL) {
edata_list_active_remove(&results, edata);
assert(nsuccess > 0);
nsuccess--;
}
if (nsuccess > 0) {
assert(sec_size_supported(&shard->sec, size));
sec_fill(tsdn, &shard->sec, size, &results, nsuccess);
/* Unlikely rollback in case of overfill */
if (!edata_list_active_empty(&results)) {
hpa_dalloc_batch(
tsdn, self, &results, deferred_work_generated);
}
}
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
return edata;
}
@ -996,10 +1007,19 @@ static void
hpa_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
assert(!edata_guarded_get(edata));
/* Just a dalloc_batch of size 1; this lets us share logic. */
edata_list_active_t dalloc_list;
edata_list_active_init(&dalloc_list);
edata_list_active_append(&dalloc_list, edata);
hpa_shard_t *shard = hpa_from_pai(self);
sec_dalloc(tsdn, &shard->sec, &dalloc_list);
if (edata_list_active_empty(&dalloc_list)) {
/* sec consumed the pointer */
*deferred_work_generated = false;
return;
}
/* We may have more than one pointer to flush now */
hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
}
@ -1063,15 +1083,32 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
return time_ns;
}
static void
hpa_sec_flush_impl(tsdn_t *tsdn, hpa_shard_t *shard) {
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
sec_flush(tsdn, &shard->sec, &to_flush);
bool deferred_work_generated;
hpa_dalloc_batch(
tsdn, (pai_t *)shard, &to_flush, &deferred_work_generated);
}
void
hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
hpa_sec_flush_impl(tsdn, shard);
malloc_mutex_lock(tsdn, &shard->mtx);
edata_cache_fast_disable(tsdn, &shard->ecf);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
void
hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_sec_flush_impl(tsdn, shard);
}
static void
hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
assert(bin_stats->npageslabs == 0);
@ -1093,6 +1130,7 @@ hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
void
hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
hpa_shard_flush(tsdn, shard);
/*
* By the time we're here, the arena code should have dalloc'd all the
* active extents, which means we should have eventually evicted
@ -1137,6 +1175,12 @@ hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_unlock(tsdn, &shard->mtx);
}
void
hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
sec_prefork2(tsdn, &shard->sec);
}
void
hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
@ -1155,6 +1199,7 @@ void
hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
sec_postfork_parent(tsdn, &shard->sec);
malloc_mutex_postfork_parent(tsdn, &shard->grow_mtx);
malloc_mutex_postfork_parent(tsdn, &shard->mtx);
}
@ -1163,6 +1208,7 @@ void
hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
sec_postfork_child(tsdn, &shard->sec);
malloc_mutex_postfork_child(tsdn, &shard->grow_mtx);
malloc_mutex_postfork_child(tsdn, &shard->mtx);
}

17
src/jemalloc.c
View file

@ -1013,6 +1013,15 @@ malloc_conf_error(
/* However, tolerate experimental features. */
return;
}
const char *deprecated[] = {"hpa_sec_bytes_after_flush"};
const size_t deprecated_cnt = (sizeof(deprecated)
/ sizeof(deprecated[0]));
for (size_t i = 0; i < deprecated_cnt; ++i) {
if (strncmp(k, deprecated[i], strlen(deprecated[i])) == 0) {
/* Tolerate deprecated features. */
return;
}
}
had_conf_error = true;
}
@ -1685,7 +1694,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
}
CONF_CONTINUE;
}
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards,
"hpa_sec_nshards", 0, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
@ -1694,13 +1702,10 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN,
CONF_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
"hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush,
"hpa_sec_bytes_after_flush", PAGE, 0,
"hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
"hpa_sec_batch_fill_extra", 0, HUGEPAGE_PAGES,
"hpa_sec_batch_fill_extra", 1, HUGEPAGE_PAGES,
CONF_CHECK_MIN, CONF_CHECK_MAX, true);
if (CONF_MATCH("slab_sizes")) {

22
src/pa.c
View file

@ -67,12 +67,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
bool
pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap,
shard->base, &shard->edata_cache, shard->ind, hpa_opts)) {
return true;
}
if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,
hpa_sec_opts)) {
if (hpa_shard_init(tsdn, &shard->hpa_shard, &shard->central->hpa,
shard->emap, shard->base, &shard->edata_cache, shard->ind,
hpa_opts, hpa_sec_opts)) {
return true;
}
shard->ever_used_hpa = true;
@ -85,7 +82,6 @@ void
pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
if (shard->ever_used_hpa) {
sec_disable(tsdn, &shard->hpa_sec);
hpa_shard_disable(tsdn, &shard->hpa_shard);
}
}
@ -93,8 +89,13 @@ pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
void
pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
pa_shard_flush(tsdn, shard);
}
void
pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
if (shard->ever_used_hpa) {
sec_flush(tsdn, &shard->hpa_sec);
hpa_shard_flush(tsdn, &shard->hpa_shard);
}
}
@ -107,7 +108,6 @@ void
pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
pac_destroy(tsdn, &shard->pac);
if (shard->ever_used_hpa) {
sec_flush(tsdn, &shard->hpa_sec);
hpa_shard_destroy(tsdn, &shard->hpa_shard);
}
}
@ -115,7 +115,7 @@ pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
static pai_t *
pa_get_pai(pa_shard_t *shard, edata_t *edata) {
return (edata_pai_get(edata) == EXTENT_PAI_PAC ? &shard->pac.pai
: &shard->hpa_sec.pai);
: &shard->hpa_shard.pai);
}
edata_t *
@ -128,7 +128,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
edata_t *edata = NULL;
if (!guarded && pa_shard_uses_hpa(shard)) {
edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment,
edata = pai_alloc(tsdn, &shard->hpa_shard.pai, size, alignment,
zero, /* guarded */ false, slab, deferred_work_generated);
}
/*

10
src/pa_extra.c
View file

@ -17,7 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
void
pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
if (shard->ever_used_hpa) {
sec_prefork2(tsdn, &shard->hpa_sec);
hpa_shard_prefork2(tsdn, &shard->hpa_shard);
}
}
@ -54,7 +54,6 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
sec_postfork_parent(tsdn, &shard->hpa_sec);
hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
}
}
@ -69,7 +68,6 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
sec_postfork_child(tsdn, &shard->hpa_sec);
hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
}
}
@ -104,8 +102,7 @@ pa_shard_basic_stats_merge(
void
pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
size_t *resident) {
hpa_shard_stats_t *hpa_stats_out, size_t *resident) {
cassert(config_stats);
pa_shard_stats_out->pac_stats.retained +=
@ -170,7 +167,6 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
if (shard->ever_used_hpa) {
hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out);
}
}
@ -204,7 +200,7 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->hpa_shard.grow_mtx,
arena_prof_mutex_hpa_shard_grow);
sec_mutex_stats_read(tsdn, &shard->hpa_sec,
sec_mutex_stats_read(tsdn, &shard->hpa_shard.sec,
&mutex_prof_data[arena_prof_mutex_hpa_sec]);
}
}

8
src/pac.c
View file

@ -97,11 +97,9 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
pac->pai.alloc = &pac_alloc_impl;
pac->pai.alloc_batch = &pai_alloc_batch_default;
pac->pai.expand = &pac_expand_impl;
pac->pai.shrink = &pac_shrink_impl;
pac->pai.dalloc = &pac_dalloc_impl;
pac->pai.dalloc_batch = &pai_dalloc_batch_default;
pac->pai.time_until_deferred_work = &pac_time_until_deferred_work;
return false;
@ -449,8 +447,8 @@ decay_with_process_madvise(edata_list_inactive_t *decay_extents) {
size_t cur = 0, total_bytes = 0;
for (edata_t *edata = edata_list_inactive_first(decay_extents);
edata != NULL;
edata = edata_list_inactive_next(decay_extents, edata)) {
edata != NULL;
edata = edata_list_inactive_next(decay_extents, edata)) {
size_t pages_bytes = edata_size_get(edata);
vec[cur].iov_base = edata_base_get(edata);
vec[cur].iov_len = pages_bytes;
@ -511,7 +509,7 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
}
for (edata_t *edata = edata_list_inactive_first(decay_extents);
edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
edata_list_inactive_remove(decay_extents, edata);
size_t size = edata_size_get(edata);

32
src/pai.c
View file

@ -1,32 +0,0 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
size_t
pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated) {
for (size_t i = 0; i < nallocs; i++) {
bool deferred_by_alloc = false;
edata_t *edata = pai_alloc(tsdn, self, size, PAGE,
/* zero */ false, /* guarded */ false, frequent_reuse,
&deferred_by_alloc);
*deferred_work_generated |= deferred_by_alloc;
if (edata == NULL) {
return i;
}
edata_list_active_append(results, edata);
}
return nallocs;
}
void
pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
bool *deferred_work_generated) {
edata_t *edata;
while ((edata = edata_list_active_first(list)) != NULL) {
bool deferred_by_dalloc = false;
edata_list_active_remove(list, edata);
pai_dalloc(tsdn, self, edata, &deferred_by_dalloc);
*deferred_work_generated |= deferred_by_dalloc;
}
}

564
src/sec.c
View file

@ -4,95 +4,56 @@
#include "jemalloc/internal/sec.h"
#include "jemalloc/internal/jemalloc_probe.h"
static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void sec_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
static void
static bool
sec_bin_init(sec_bin_t *bin) {
bin->being_batch_filled = false;
bin->bytes_cur = 0;
sec_bin_stats_init(&bin->stats);
edata_list_active_init(&bin->freelist);
bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN,
malloc_mutex_rank_exclusive);
if (err) {
return true;
}
return false;
}
bool
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
const sec_opts_t *opts) {
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
sec->opts = *opts;
if (opts->nshards == 0) {
return false;
}
assert(opts->max_alloc >= PAGE);
/*
* Same as tcache, sec do not cache allocs/dallocs larger than
* USIZE_GROW_SLOW_THRESHOLD because the usize above this increases
* by PAGE and the number of usizes is too large.
*/
assert(!sz_large_size_classes_disabled()
|| opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
assert(opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
size_t max_alloc = PAGE_FLOOR(opts->max_alloc);
pszind_t npsizes = sz_psz2ind(max_alloc) + 1;
size_t sz_shards = opts->nshards * sizeof(sec_shard_t);
size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t);
size_t sz_alloc = sz_shards + sz_bins;
void *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE);
size_t ntotal_bins = opts->nshards * (size_t)npsizes;
size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
void *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
if (dynalloc == NULL) {
return true;
}
sec_shard_t *shard_cur = (sec_shard_t *)dynalloc;
sec->shards = shard_cur;
sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards];
/* Just for asserts, below. */
sec_bin_t *bin_start = bin_cur;
for (size_t i = 0; i < opts->nshards; i++) {
sec_shard_t *shard = shard_cur;
shard_cur++;
bool err = malloc_mutex_init(&shard->mtx, "sec_shard",
WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive);
if (err) {
sec->bins = (sec_bin_t *)dynalloc;
for (pszind_t j = 0; j < ntotal_bins; j++) {
if (sec_bin_init(&sec->bins[j])) {
return true;
}
shard->enabled = true;
shard->bins = bin_cur;
for (pszind_t j = 0; j < npsizes; j++) {
sec_bin_init(&shard->bins[j]);
bin_cur++;
}
shard->bytes_cur = 0;
shard->to_flush_next = 0;
}
/*
* Should have exactly matched the bin_start to the first unused byte
* after the shards.
*/
assert((void *)shard_cur == (void *)bin_start);
/* And the last bin to use up the last bytes of the allocation. */
assert((char *)bin_cur == ((char *)dynalloc + sz_alloc));
sec->fallback = fallback;
sec->opts = *opts;
sec->npsizes = npsizes;
/*
* Initialize these last so that an improper use of an SEC whose
* initialization failed will segfault in an easy-to-spot way.
*/
sec->pai.alloc = &sec_alloc;
sec->pai.alloc_batch = &pai_alloc_batch_default;
sec->pai.expand = &sec_expand;
sec->pai.shrink = &sec_shrink;
sec->pai.dalloc = &sec_dalloc;
sec->pai.dalloc_batch = &pai_dalloc_batch_default;
return false;
}
static sec_shard_t *
static uint8_t
sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
/*
* Eventually, we should implement affinity, tracking source shard using
@ -100,7 +61,7 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
* distribute across all shards.
*/
if (tsdn_null(tsdn)) {
return &sec->shards[0];
return 0;
}
tsd_t *tsd = tsdn_tsd(tsdn);
uint8_t *idxp = tsd_sec_shardp_get(tsd);
@ -118,284 +79,252 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
assert(idx < (uint32_t)sec->opts.nshards);
*idxp = (uint8_t)idx;
}
return &sec->shards[*idxp];
return *idxp;
}
/*
* Perhaps surprisingly, this can be called on the alloc pathways; if we hit an
* empty cache, we'll try to fill it, which can push the shard over it's limit.
*/
static void
sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
while (shard->bytes_cur > sec->opts.bytes_after_flush) {
/* Pick a victim. */
sec_bin_t *bin = &shard->bins[shard->to_flush_next];
/* Update our victim-picking state. */
shard->to_flush_next++;
if (shard->to_flush_next == sec->npsizes) {
shard->to_flush_next = 0;
}
assert(shard->bytes_cur >= bin->bytes_cur);
if (bin->bytes_cur != 0) {
shard->bytes_cur -= bin->bytes_cur;
bin->bytes_cur = 0;
edata_list_active_concat(&to_flush, &bin->freelist);
}
/*
* Either bin->bytes_cur was 0, in which case we didn't touch
* the bin list but it should be empty anyways (or else we
* missed a bytes_cur update on a list modification), or it
* *was* 0 and we emptied it ourselves. Either way, it should
* be empty now.
*/
assert(edata_list_active_empty(&bin->freelist));
}
malloc_mutex_unlock(tsdn, &shard->mtx);
bool deferred_work_generated = false;
pai_dalloc_batch(
tsdn, sec->fallback, &to_flush, &deferred_work_generated);
static sec_bin_t *
sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) {
assert(shard < sec->opts.nshards);
size_t ind = (size_t)shard * sec->npsizes + pszind;
assert(ind < sec->npsizes * sec->opts.nshards);
return &sec->bins[ind];
}
static edata_t *
sec_shard_alloc_locked(
tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, sec_bin_t *bin) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
if (!shard->enabled) {
return NULL;
}
sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
malloc_mutex_assert_owner(tsdn, &bin->mtx);
edata_t *edata = edata_list_active_first(&bin->freelist);
if (edata != NULL) {
assert(!edata_list_active_empty(&bin->freelist));
edata_list_active_remove(&bin->freelist, edata);
assert(edata_size_get(edata) <= bin->bytes_cur);
bin->bytes_cur -= edata_size_get(edata);
assert(edata_size_get(edata) <= shard->bytes_cur);
shard->bytes_cur -= edata_size_get(edata);
size_t sz = edata_size_get(edata);
assert(sz <= bin->bytes_cur && sz > 0);
bin->bytes_cur -= sz;
bin->stats.nhits++;
}
return edata;
}
static edata_t *
sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
sec_bin_t *bin, size_t size, bool frequent_reuse) {
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
sec_multishard_trylock_alloc(
tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
assert(sec->opts.nshards > 0);
edata_list_active_t result;
edata_list_active_init(&result);
bool deferred_work_generated = false;
size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size,
1 + sec->opts.batch_fill_extra, &result, frequent_reuse,
&deferred_work_generated);
edata_t *ret = edata_list_active_first(&result);
if (ret != NULL) {
edata_list_active_remove(&result, ret);
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
sec_bin_t *bin;
for (size_t i = 0; i < sec->opts.nshards; ++i) {
bin = sec_bin_pick(sec, cur_shard, pszind);
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
edata_t *edata = sec_bin_alloc_locked(
tsdn, sec, bin, size);
malloc_mutex_unlock(tsdn, &bin->mtx);
if (edata != NULL) {
JE_USDT(sec_alloc, 5, sec, bin, edata, size,
/* frequent_reuse */ 1);
return edata;
}
}
cur_shard++;
if (cur_shard == sec->opts.nshards) {
cur_shard = 0;
}
}
malloc_mutex_lock(tsdn, &shard->mtx);
bin->being_batch_filled = false;
/*
* Handle the easy case first: nothing to cache. Note that this can
* only happen in case of OOM, since sec_alloc checks the expected
* number of allocs, and doesn't bother going down the batch_fill
* pathway if there won't be anything left to cache. So to be in this
* code path, we must have asked for > 1 alloc, but only gotten 1 back.
*/
if (nalloc <= 1) {
malloc_mutex_unlock(tsdn, &shard->mtx);
return ret;
/* No bin had alloc or had the extent */
assert(cur_shard == sec_shard_pick(tsdn, sec));
bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
if (edata == NULL) {
/* Only now we know it is a miss */
bin->stats.nmisses++;
}
size_t new_cached_bytes = (nalloc - 1) * size;
edata_list_active_concat(&bin->freelist, &result);
bin->bytes_cur += new_cached_bytes;
shard->bytes_cur += new_cached_bytes;
if (shard->bytes_cur > sec->opts.max_bytes) {
sec_flush_some_and_unlock(tsdn, sec, shard);
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
}
return ret;
malloc_mutex_unlock(tsdn, &bin->mtx);
JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1);
return edata;
}
static edata_t *
sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
edata_t *
sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
if (!sec_size_supported(sec, size)) {
return NULL;
}
assert((size & PAGE_MASK) == 0);
assert(!guarded);
sec_t *sec = (sec_t *)self;
if (zero || alignment > PAGE || sec->opts.nshards == 0
|| size > sec->opts.max_alloc) {
return pai_alloc(tsdn, sec->fallback, size, alignment, zero,
/* guarded */ false, frequent_reuse,
deferred_work_generated);
}
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
sec_bin_t *bin = &shard->bins[pszind];
bool do_batch_fill = false;
malloc_mutex_lock(tsdn, &shard->mtx);
edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin);
if (edata == NULL) {
if (!bin->being_batch_filled
&& sec->opts.batch_fill_extra > 0) {
bin->being_batch_filled = true;
do_batch_fill = true;
/*
* If there's only one shard, skip the trylock optimization and
* go straight to the blocking lock.
*/
if (sec->opts.nshards == 1) {
sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
if (edata == NULL) {
bin->stats.nmisses++;
}
malloc_mutex_unlock(tsdn, &bin->mtx);
JE_USDT(sec_alloc, 5, sec, bin, edata, size,
/* frequent_reuse */ 1);
return edata;
}
malloc_mutex_unlock(tsdn, &shard->mtx);
if (edata == NULL) {
if (do_batch_fill) {
edata = sec_batch_fill_and_alloc(
tsdn, sec, shard, bin, size, frequent_reuse);
} else {
edata = pai_alloc(tsdn, sec->fallback, size, alignment,
zero, /* guarded */ false, frequent_reuse,
deferred_work_generated);
}
}
JE_USDT(sec_alloc, 5, sec, shard, edata, size, frequent_reuse);
return edata;
}
static bool
sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
JE_USDT(sec_expand, 4, sec, edata, old_size, new_size);
return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero,
deferred_work_generated);
}
static bool
sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
JE_USDT(sec_shrink, 4, sec, edata, old_size, new_size);
return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size,
deferred_work_generated);
return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
}
static void
sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
shard->bytes_cur = 0;
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
for (pszind_t i = 0; i < sec->npsizes; i++) {
sec_bin_t *bin = &shard->bins[i];
bin->bytes_cur = 0;
edata_list_active_concat(&to_flush, &bin->freelist);
}
sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
edata_list_active_t *dalloc_list) {
malloc_mutex_assert_owner(tsdn, &bin->mtx);
/*
* Ordinarily we would try to avoid doing the batch deallocation while
* holding the shard mutex, but the flush_all pathways only happen when
* we're disabling the HPA or resetting the arena, both of which are
* rare pathways.
*/
bool deferred_work_generated = false;
pai_dalloc_batch(
tsdn, sec->fallback, &to_flush, &deferred_work_generated);
}
static void
sec_shard_dalloc_and_unlock(
tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_t *edata) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
assert(shard->bytes_cur <= sec->opts.max_bytes);
size_t size = edata_size_get(edata);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
/*
* Prepending here results in LIFO allocation per bin, which seems
* reasonable.
*/
sec_bin_t *bin = &shard->bins[pszind];
edata_list_active_prepend(&bin->freelist, edata);
bin->bytes_cur += size;
shard->bytes_cur += size;
if (shard->bytes_cur > sec->opts.max_bytes) {
/*
* We've exceeded the shard limit. We make two nods in the
* direction of fragmentation avoidance: we flush everything in
* the shard, rather than one particular bin, and we hold the
* lock while flushing (in case one of the extents we flush is
* highly preferred from a fragmentation-avoidance perspective
* in the backing allocator). This has the extra advantage of
* not requiring advanced cache balancing strategies.
*/
sec_flush_some_and_unlock(tsdn, sec, shard);
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
}
}
edata_t *edata = edata_list_active_first(dalloc_list);
assert(edata != NULL);
edata_list_active_remove(dalloc_list, edata);
JE_USDT(sec_dalloc, 3, sec, bin, edata);
edata_list_active_prepend(&bin->freelist, edata);
/* Single extent can be returned to SEC */
assert(edata_list_active_empty(dalloc_list));
static void
sec_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
if (sec->opts.nshards == 0
|| edata_size_get(edata) > sec->opts.max_alloc) {
pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
if (bin->bytes_cur <= sec->opts.max_bytes) {
bin->stats.ndalloc_noflush++;
return;
}
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
JE_USDT(sec_dalloc, 3, sec, shard, edata);
malloc_mutex_lock(tsdn, &shard->mtx);
if (shard->enabled) {
sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata);
bin->stats.ndalloc_flush++;
/* we want to flush 1/4 of max_bytes */
size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2);
while (bin->bytes_cur > bytes_target
&& !edata_list_active_empty(&bin->freelist)) {
edata_t *cur = edata_list_active_last(&bin->freelist);
size_t sz = edata_size_get(cur);
assert(sz <= bin->bytes_cur && sz > 0);
bin->bytes_cur -= sz;
edata_list_active_remove(&bin->freelist, cur);
edata_list_active_append(dalloc_list, cur);
}
}
static void
sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
pszind_t pszind, edata_list_active_t *dalloc_list) {
assert(sec->opts.nshards > 0);
/* Try to dalloc in this threads bin first */
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
for (size_t i = 0; i < sec->opts.nshards; ++i) {
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
sec_bin_dalloc_locked(
tsdn, sec, bin, size, dalloc_list);
malloc_mutex_unlock(tsdn, &bin->mtx);
return;
}
cur_shard++;
if (cur_shard == sec->opts.nshards) {
cur_shard = 0;
}
}
/* No bin had alloc or had the extent */
assert(cur_shard == sec_shard_pick(tsdn, sec));
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
void
sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
if (!sec_is_used(sec)) {
return;
}
edata_t *edata = edata_list_active_first(dalloc_list);
size_t size = edata_size_get(edata);
if (size > sec->opts.max_alloc) {
return;
}
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
/*
* If there's only one shard, skip the trylock optimization and
* go straight to the blocking lock.
*/
if (sec->opts.nshards == 1) {
sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
malloc_mutex_unlock(tsdn, &bin->mtx);
return;
}
sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
}
void
sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
size_t nallocs) {
assert((size & PAGE_MASK) == 0);
assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
assert(nallocs > 0);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
malloc_mutex_lock(tsdn, &bin->mtx);
size_t new_cached_bytes = nallocs * size;
if (bin->bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
assert(!edata_list_active_empty(result));
edata_list_active_concat(&bin->freelist, result);
bin->bytes_cur += new_cached_bytes;
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
/*
* Unlikely case of many threads filling at the same time and
* going above max.
*/
bin->stats.noverfills++;
while (bin->bytes_cur + size <= sec->opts.max_bytes) {
edata_t *edata = edata_list_active_first(result);
if (edata == NULL) {
break;
}
edata_list_active_remove(result, edata);
assert(size == edata_size_get(edata));
edata_list_active_append(&bin->freelist, edata);
bin->bytes_cur += size;
}
}
malloc_mutex_unlock(tsdn, &bin->mtx);
}
void
sec_flush(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
if (!sec_is_used(sec)) {
return;
}
}
void
sec_disable(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sec->shards[i].enabled = false;
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
bin->bytes_cur = 0;
edata_list_active_concat(to_flush, &bin->freelist);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
}
void
sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
if (!sec_is_used(sec)) {
return;
}
size_t sum = 0;
for (size_t i = 0; i < sec->opts.nshards; i++) {
/*
* We could save these lock acquisitions by making bytes_cur
* atomic, but stats collection is rare anyways and we expect
* the number and type of stats to get more interesting.
*/
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sum += sec->shards[i].bytes_cur;
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
sum += bin->bytes_cur;
sec_bin_stats_accum(&stats->total, &bin->stats);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
stats->bytes += sum;
}
@ -403,31 +332,50 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
void
sec_mutex_stats_read(
tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
malloc_mutex_prof_accum(
tsdn, mutex_prof_data, &sec->shards[i].mtx);
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
malloc_mutex_prof_accum(tsdn, mutex_prof_data, &bin->mtx);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
}
void
sec_prefork2(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_prefork(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_prefork(tsdn, &bin->mtx);
}
}
void
sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_postfork_parent(tsdn, &bin->mtx);
}
}
void
sec_postfork_child(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_postfork_child(tsdn, &bin->mtx);
}
}

27
src/stats.c
View file

@ -791,9 +791,35 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
static void
stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) {
size_t sec_bytes;
size_t sec_hits;
size_t sec_misses;
size_t sec_dalloc_flush;
size_t sec_dalloc_noflush;
size_t sec_overfills;
CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t);
emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache",
emitter_type_size, &sec_bytes);
CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec_hits, size_t);
emitter_kv(emitter, "sec_hits", "Total hits in small extent cache",
emitter_type_size, &sec_hits);
CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec_misses, size_t);
emitter_kv(emitter, "sec_misses", "Total misses in small extent cache",
emitter_type_size, &sec_misses);
CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i,
&sec_dalloc_noflush, size_t);
emitter_kv(emitter, "sec_dalloc_noflush",
"Dalloc calls without flush in small extent cache",
emitter_type_size, &sec_dalloc_noflush);
CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, &sec_dalloc_flush,
size_t);
emitter_kv(emitter, "sec_dalloc_flush",
"Dalloc calls with flush in small extent cache", emitter_type_size,
&sec_dalloc_flush);
CTL_M2_GET(
"stats.arenas.0.hpa_sec_overfills", i, &sec_overfills, size_t);
emitter_kv(emitter, "sec_overfills",
"sec_fill calls that went over max_bytes", emitter_type_size,
&sec_overfills);
}
static void
@ -1642,7 +1668,6 @@ stats_general_print(emitter_t *emitter) {
OPT_WRITE_SIZE_T("hpa_sec_nshards")
OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush")
OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
OPT_WRITE_BOOL("huge_arena_pac_thp")
OPT_WRITE_CHAR_P("metadata_thp")