mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-07-26 10:37:19 +03:00
Shirui Cheng
Guangli Dai
parent
ced8b3cffb
commit
2114349a4e
30 changed files with 124 additions and 1364 deletions
89
src/arena.c
89
src/arena.c
|
|
@ -39,7 +39,8 @@ div_info_t arena_binind_div_info[SC_NBINS];
|
|||
size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
|
||||
size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT;
|
||||
|
||||
uint32_t arena_bin_offsets[SC_NBINS];
|
||||
uint32_t arena_bin_offsets[SC_NBINS];
|
||||
static unsigned nbins_total;
|
||||
|
||||
/*
|
||||
* a0 is used to handle huge requests before malloc init completes. After
|
||||
|
|
@ -674,17 +675,11 @@ arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, edata_t *slab) {
|
|||
}
|
||||
|
||||
static void
|
||||
arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin, unsigned binind) {
|
||||
arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) {
|
||||
edata_t *slab;
|
||||
|
||||
malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock);
|
||||
|
||||
if (arena_bin_has_batch(binind)) {
|
||||
bin_with_batch_t *batched_bin = (bin_with_batch_t *)bin;
|
||||
batcher_init(
|
||||
&batched_bin->remote_frees, BIN_REMOTE_FREE_ELEMS_MAX);
|
||||
}
|
||||
|
||||
if (bin->slabcur != NULL) {
|
||||
slab = bin->slabcur;
|
||||
bin->slabcur = NULL;
|
||||
|
|
@ -835,8 +830,7 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
|
|||
/* Bins. */
|
||||
for (unsigned i = 0; i < SC_NBINS; i++) {
|
||||
for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
|
||||
arena_bin_reset(
|
||||
tsd, arena, arena_get_bin(arena, i, j), i);
|
||||
arena_bin_reset(tsd, arena, arena_get_bin(arena, i, j));
|
||||
}
|
||||
}
|
||||
pa_shard_reset(tsd_tsdn(tsd), &arena->pa_shard);
|
||||
|
|
@ -1103,19 +1097,8 @@ arena_cache_bin_fill_small(tsdn_t *tsdn, arena_t *arena, cache_bin_t *cache_bin,
|
|||
unsigned binshard;
|
||||
bin_t *bin = arena_bin_choose(tsdn, arena, binind, &binshard);
|
||||
|
||||
/*
|
||||
* This has some fields that are conditionally initialized down batch
|
||||
* flush pathways. This can trigger static analysis warnings deeper
|
||||
* down in the static. The accesses are guarded by the same checks as
|
||||
* the initialization, but the analysis isn't able to track that across
|
||||
* multiple stack frames.
|
||||
*/
|
||||
arena_bin_flush_batch_state_t batch_flush_state
|
||||
JEMALLOC_CLANG_ANALYZER_SILENCE_INIT({0});
|
||||
label_refill:
|
||||
malloc_mutex_lock(tsdn, &bin->lock);
|
||||
arena_bin_flush_batch_after_lock(
|
||||
tsdn, arena, bin, binind, &batch_flush_state);
|
||||
|
||||
while (filled < nfill_min) {
|
||||
/* Try batch-fill from slabcur first. */
|
||||
|
|
@ -1176,11 +1159,7 @@ label_refill:
|
|||
cache_bin->tstats.nrequests = 0;
|
||||
}
|
||||
|
||||
arena_bin_flush_batch_before_unlock(
|
||||
tsdn, arena, bin, binind, &batch_flush_state);
|
||||
malloc_mutex_unlock(tsdn, &bin->lock);
|
||||
arena_bin_flush_batch_after_unlock(
|
||||
tsdn, arena, bin, binind, &batch_flush_state);
|
||||
|
||||
if (alloc_and_retry) {
|
||||
assert(fresh_slab == NULL);
|
||||
|
|
@ -1474,16 +1453,12 @@ arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) {
|
|||
malloc_mutex_lock(tsdn, &bin->lock);
|
||||
arena_dalloc_bin_locked_info_t info;
|
||||
arena_dalloc_bin_locked_begin(&info, binind);
|
||||
edata_t *dalloc_slabs[1];
|
||||
unsigned dalloc_slabs_count = 0;
|
||||
arena_dalloc_bin_locked_step(tsdn, arena, bin, &info, binind, edata,
|
||||
ptr, dalloc_slabs, /* ndalloc_slabs */ 1, &dalloc_slabs_count,
|
||||
/* dalloc_slabs_extra */ NULL);
|
||||
bool ret = arena_dalloc_bin_locked_step(
|
||||
tsdn, arena, bin, &info, binind, edata, ptr);
|
||||
arena_dalloc_bin_locked_finish(tsdn, arena, bin, &info);
|
||||
malloc_mutex_unlock(tsdn, &bin->lock);
|
||||
|
||||
if (dalloc_slabs_count != 0) {
|
||||
assert(dalloc_slabs[0] == edata);
|
||||
if (ret) {
|
||||
arena_slab_dalloc(tsdn, arena, edata);
|
||||
}
|
||||
}
|
||||
|
|
@ -1722,6 +1697,7 @@ arena_t *
|
|||
arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
|
||||
arena_t *arena;
|
||||
base_t *base;
|
||||
unsigned i;
|
||||
|
||||
if (ind == 0) {
|
||||
base = b0get();
|
||||
|
|
@ -1734,13 +1710,14 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
|
|||
}
|
||||
|
||||
size_t arena_size = ALIGNMENT_CEILING(sizeof(arena_t), CACHELINE)
|
||||
+ sizeof(bin_with_batch_t) * bin_info_nbatched_bins
|
||||
+ sizeof(bin_t) * bin_info_nunbatched_bins;
|
||||
+ sizeof(bin_t) * nbins_total;
|
||||
arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE);
|
||||
if (arena == NULL) {
|
||||
goto label_error;
|
||||
}
|
||||
|
||||
JEMALLOC_SUPPRESS_WARN_ON_USAGE(
|
||||
assert((uintptr_t)&arena->all_bins[nbins_total - 1] + sizeof(bin_t)
|
||||
<= (uintptr_t)arena + arena_size);)
|
||||
atomic_store_u(&arena->nthreads[0], 0, ATOMIC_RELAXED);
|
||||
atomic_store_u(&arena->nthreads[1], 0, ATOMIC_RELAXED);
|
||||
arena->last_thd = NULL;
|
||||
|
|
@ -1779,13 +1756,11 @@ arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config) {
|
|||
|
||||
/* Initialize bins. */
|
||||
atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE);
|
||||
for (unsigned i = 0; i < SC_NBINS; i++) {
|
||||
for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
|
||||
bin_t *bin = arena_get_bin(arena, i, j);
|
||||
bool err = bin_init(bin, i);
|
||||
if (err) {
|
||||
goto label_error;
|
||||
}
|
||||
for (i = 0; i < nbins_total; i++) {
|
||||
JEMALLOC_SUPPRESS_WARN_ON_USAGE(
|
||||
bool err = bin_init(&arena->all_bins[i]);)
|
||||
if (err) {
|
||||
goto label_error;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1943,10 +1918,8 @@ arena_boot(sc_data_t *sc_data, base_t *base, bool hpa) {
|
|||
uint32_t cur_offset = (uint32_t)offsetof(arena_t, all_bins);)
|
||||
for (szind_t i = 0; i < SC_NBINS; i++) {
|
||||
arena_bin_offsets[i] = cur_offset;
|
||||
uint32_t bin_sz = (i < bin_info_nbatched_sizes
|
||||
? sizeof(bin_with_batch_t)
|
||||
: sizeof(bin_t));
|
||||
cur_offset += (uint32_t)bin_infos[i].n_shards * bin_sz;
|
||||
nbins_total += bin_infos[i].n_shards;
|
||||
cur_offset += (uint32_t)(bin_infos[i].n_shards * sizeof(bin_t));
|
||||
}
|
||||
return pa_central_init(
|
||||
&arena_pa_central_global, base, hpa, &hpa_hooks_default);
|
||||
|
|
@ -1996,21 +1969,17 @@ arena_prefork7(tsdn_t *tsdn, arena_t *arena) {
|
|||
|
||||
void
|
||||
arena_prefork8(tsdn_t *tsdn, arena_t *arena) {
|
||||
for (szind_t i = 0; i < SC_NBINS; i++) {
|
||||
for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
|
||||
bin_t *bin = arena_get_bin(arena, i, j);
|
||||
bin_prefork(tsdn, bin, arena_bin_has_batch(i));
|
||||
}
|
||||
for (unsigned i = 0; i < nbins_total; i++) {
|
||||
JEMALLOC_SUPPRESS_WARN_ON_USAGE(
|
||||
bin_prefork(tsdn, &arena->all_bins[i]);)
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) {
|
||||
for (szind_t i = 0; i < SC_NBINS; i++) {
|
||||
for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
|
||||
bin_t *bin = arena_get_bin(arena, i, j);
|
||||
bin_postfork_parent(tsdn, bin, arena_bin_has_batch(i));
|
||||
}
|
||||
for (unsigned i = 0; i < nbins_total; i++) {
|
||||
JEMALLOC_SUPPRESS_WARN_ON_USAGE(
|
||||
bin_postfork_parent(tsdn, &arena->all_bins[i]);)
|
||||
}
|
||||
|
||||
malloc_mutex_postfork_parent(tsdn, &arena->large_mtx);
|
||||
|
|
@ -2047,11 +2016,9 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) {
|
|||
}
|
||||
}
|
||||
|
||||
for (szind_t i = 0; i < SC_NBINS; i++) {
|
||||
for (unsigned j = 0; j < bin_infos[i].n_shards; j++) {
|
||||
bin_t *bin = arena_get_bin(arena, i, j);
|
||||
bin_postfork_child(tsdn, bin, arena_bin_has_batch(i));
|
||||
}
|
||||
for (unsigned i = 0; i < nbins_total; i++) {
|
||||
JEMALLOC_SUPPRESS_WARN_ON_USAGE(
|
||||
bin_postfork_child(tsdn, &arena->all_bins[i]);)
|
||||
}
|
||||
|
||||
malloc_mutex_postfork_child(tsdn, &arena->large_mtx);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue