romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-04-14 22:51:50 +03:00
Code Issues Projects Releases Packages Wiki Activity

Add mallctl to set and get ncached_max of each cache_bin.

Browse source 
1. `thread_tcache_ncached_max_read_sizeclass` allows users to get the
    ncached_max of the bin with the input sizeclass, passed in through
    oldp (will be upper casted if not an exact bin size is given).
2. `thread_tcache_ncached_max_write` takes in a char array
    representing the settings for bins in the tcache.
This commit is contained in:
guangli-dai 2023-09-19 14:37:09 -07:00 • committed by Qi Wang
parent 6b197fdd46
commit 630f7de952
14 changed files with 477 additions and 70 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/cache_bin.c
View file

@ -10,8 +10,9 @@ const uintptr_t disabled_bin = JUNK_ADDR;
void
cache_bin_info_init(cache_bin_info_t *info,
cache_bin_sz_t ncached_max) {
assert(ncached_max <= CACHE_BIN_NCACHED_MAX);
size_t stack_size = (size_t)ncached_max * sizeof(void *);
assert(stack_size < ((size_t)1 << (sizeof(cache_bin_sz_t) * 8)));
assert(stack_size <= UINT16_MAX);
info->ncached_max = (cache_bin_sz_t)ncached_max;
}

85
src/ctl.c
View file

@ -68,6 +68,8 @@ CTL_PROTO(max_background_threads)
CTL_PROTO(thread_tcache_enabled)
CTL_PROTO(thread_tcache_max)
CTL_PROTO(thread_tcache_flush)
CTL_PROTO(thread_tcache_ncached_max_write)
CTL_PROTO(thread_tcache_ncached_max_read_sizeclass)
CTL_PROTO(thread_peak_read)
CTL_PROTO(thread_peak_reset)
CTL_PROTO(thread_prof_name)
@ -374,10 +376,17 @@ CTL_PROTO(stats_mutexes_reset)
*/
#define INDEX(i) {false}, i##_index
static const ctl_named_node_t thread_tcache_ncached_max_node[] = {
{NAME("read_sizeclass"),
CTL(thread_tcache_ncached_max_read_sizeclass)},
{NAME("write"), CTL(thread_tcache_ncached_max_write)}
};
static const ctl_named_node_t thread_tcache_node[] = {
{NAME("enabled"), CTL(thread_tcache_enabled)},
{NAME("max"), CTL(thread_tcache_max)},
{NAME("flush"), CTL(thread_tcache_flush)}
{NAME("flush"), CTL(thread_tcache_flush)},
{NAME("ncached_max"), CHILD(named, thread_tcache_ncached_max)}
};
static const ctl_named_node_t thread_peak_node[] = {
@ -2282,6 +2291,78 @@ label_return:
CTL_RO_NL_GEN(thread_allocated, tsd_thread_allocated_get(tsd), uint64_t)
CTL_RO_NL_GEN(thread_allocatedp, tsd_thread_allocatedp_get(tsd), uint64_t *)
static int
thread_tcache_ncached_max_read_sizeclass_ctl(tsd_t *tsd, const size_t *mib,
size_t miblen, void *oldp, size_t *oldlenp, void *newp,
size_t newlen) {
int ret;
size_t bin_size = 0;
/* Read the bin size from newp. */
if (newp == NULL) {
ret = EINVAL;
goto label_return;
}
WRITE(bin_size, size_t);
cache_bin_sz_t ncached_max = 0;
if (tcache_bin_ncached_max_read(tsd, bin_size, &ncached_max)) {
ret = EINVAL;
goto label_return;
}
size_t result = (size_t)ncached_max;
READ(result, size_t);
ret = 0;
label_return:
return ret;
}
static int
thread_tcache_ncached_max_write_ctl(tsd_t *tsd, const size_t *mib,
size_t miblen, void *oldp, size_t *oldlenp, void *newp,
size_t newlen) {
int ret;
WRITEONLY();
if (newp != NULL) {
if (!tcache_available(tsd)) {
ret = ENOENT;
goto label_return;
}
char *settings = NULL;
WRITE(settings, char *);
if (settings == NULL) {
ret = EINVAL;
goto label_return;
}
/* Get the length of the setting string safely. */
char *end = (char *)memchr(settings, '\0',
CTL_MULTI_SETTING_MAX_LEN);
if (end == NULL) {
ret = EINVAL;
goto label_return;
}
/*
* Exclude the last '\0' for len since it is not handled by
* multi_setting_parse_next.
*/
size_t len = (uintptr_t)end - (uintptr_t)settings;
if (len == 0) {
ret = 0;
goto label_return;
}
if (tcache_bins_ncached_max_write(tsd, settings, len)) {
ret = EINVAL;
goto label_return;
}
}
ret = 0;
label_return:
return ret;
}
CTL_RO_NL_GEN(thread_deallocated, tsd_thread_deallocated_get(tsd), uint64_t)
CTL_RO_NL_GEN(thread_deallocatedp, tsd_thread_deallocatedp_get(tsd), uint64_t *)
@ -3155,7 +3236,7 @@ CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t)
CTL_RO_NL_GEN(arenas_page, PAGE, size_t)
CTL_RO_NL_GEN(arenas_tcache_max, global_do_not_change_tcache_maxclass, size_t)
CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned)
CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_nbins, unsigned)
CTL_RO_NL_GEN(arenas_nhbins, global_do_not_change_tcache_nbins, unsigned)
CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t)
CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t)
CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t)

49
src/jemalloc.c
View file

@ -821,50 +821,6 @@ init_opt_stats_opts(const char *v, size_t vlen, char *dest) {
assert(opts_len == strlen(dest));
}
/* Reads the next size pair in a multi-sized option. */
static bool
malloc_conf_multi_sizes_next(const char **slab_size_segment_cur,
size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) {
const char *cur = *slab_size_segment_cur;
char *end;
uintmax_t um;
set_errno(0);
/* First number, then '-' */
um = malloc_strtoumax(cur, &end, 0);
if (get_errno() != 0 || *end != '-') {
return true;
}
*slab_start = (size_t)um;
cur = end + 1;
/* Second number, then ':' */
um = malloc_strtoumax(cur, &end, 0);
if (get_errno() != 0 || *end != ':') {
return true;
}
*slab_end = (size_t)um;
cur = end + 1;
/* Last number */
um = malloc_strtoumax(cur, &end, 0);
if (get_errno() != 0) {
return true;
}
*new_size = (size_t)um;
/* Consume the separator if there is one. */
if (*end == '|') {
end++;
}
*vlen_left -= end - *slab_size_segment_cur;
*slab_size_segment_cur = end;
return false;
}
static void
malloc_conf_format_error(const char *msg, const char *begin, const char *end) {
size_t len = end - begin + 1;
@ -1351,7 +1307,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
size_t size_start;
size_t size_end;
size_t nshards;
bool err = malloc_conf_multi_sizes_next(
bool err = multi_setting_parse_next(
&bin_shards_segment_cur, &vlen_left,
&size_start, &size_end, &nshards);
if (err || bin_update_shard_size(
@ -1613,7 +1569,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
size_t slab_start;
size_t slab_end;
size_t pgs;
err = malloc_conf_multi_sizes_next(
err = multi_setting_parse_next(
&slab_size_segment_cur,
&vlen_left, &slab_start, &slab_end,
&pgs);
@ -4140,6 +4096,7 @@ batch_alloc(void **ptrs, size_t num, size_t size, int flags) {
tcache_t *tcache = tcache_get_from_ind(tsd, tcache_ind,
/* slow */ true, /* is_alloc */ true);
if (likely(tcache != NULL &&
ind < tcache_nbins_get(tcache->tcache_slow) &&
!tcache_bin_disabled(ind, &tcache->bins[ind],
tcache->tcache_slow)) && progress < batch) {
if (bin == NULL) {

83
src/tcache.c
View file

@ -63,7 +63,7 @@ unsigned opt_lg_tcache_flush_large_div = 1;
* is only used to initialize tcache_nbins in the per-thread tcache.
* Directly modifying it will not affect threads already launched.
*/
unsigned global_do_not_change_nbins;
unsigned global_do_not_change_tcache_nbins;
/*
* Max size class to be cached (can be small or large). This value is only used
* to initialize tcache_max in the per-thread tcache. Directly modifying it
@ -193,8 +193,7 @@ tcache_event(tsd_t *tsd) {
goto label_done;
}
tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind,
is_small);
tcache_bin_flush_stashed(tsd, tcache, cache_bin, szind, is_small);
cache_bin_sz_t low_water = cache_bin_low_water_get(cache_bin,
&cache_bin->bin_info);
if (low_water > 0) {
@ -591,6 +590,28 @@ tcache_bin_flush_stashed(tsd_t *tsd, tcache_t *tcache, cache_bin_t *cache_bin,
assert(head_content == *cache_bin->stack_head);
}
bool
tcache_bin_ncached_max_read(tsd_t *tsd, size_t bin_size,
cache_bin_sz_t *ncached_max) {
if (bin_size > TCACHE_MAXCLASS_LIMIT) {
return true;
}
if (!tcache_available(tsd)) {
*ncached_max = 0;
return false;
}
tcache_t *tcache = tsd_tcachep_get(tsd);
assert(tcache != NULL);
szind_t bin_ind = sz_size2index(bin_size);
cache_bin_t *bin = &tcache->bins[bin_ind];
*ncached_max = tcache_bin_disabled(bin_ind, bin, tcache->tcache_slow) ?
0: cache_bin_info_ncached_max_get(bin, &bin->bin_info);
return false;
}
void
tcache_arena_associate(tsdn_t *tsdn, tcache_slow_t *tcache_slow,
tcache_t *tcache, arena_t *arena) {
@ -651,8 +672,8 @@ static void
tcache_default_settings_init(tcache_slow_t *tcache_slow) {
assert(tcache_slow != NULL);
assert(global_do_not_change_tcache_maxclass != 0);
assert(global_do_not_change_nbins != 0);
tcache_slow->tcache_nbins = global_do_not_change_nbins;
assert(global_do_not_change_tcache_nbins != 0);
tcache_slow->tcache_nbins = global_do_not_change_tcache_nbins;
}
static void
@ -772,7 +793,7 @@ tcache_ncached_max_compute(szind_t szind) {
}
}
static void
JET_EXTERN void
tcache_bin_info_compute(cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX]) {
/*
* Compute the values for each bin, but for bins with indices larger
@ -866,7 +887,7 @@ tcache_create_explicit(tsd_t *tsd) {
* the beginning of the whole allocation (for freeing). The makes sure
* the cache bins have the requested alignment.
*/
unsigned tcache_nbins = global_do_not_change_nbins;
unsigned tcache_nbins = global_do_not_change_tcache_nbins;
size_t tcache_size, alignment;
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX] = {{0}};
tcache_bin_info_compute(tcache_bin_info);
@ -963,6 +984,52 @@ thread_tcache_max_set(tsd_t *tsd, size_t tcache_max) {
assert(tcache_nbins_get(tcache_slow) == sz_size2index(tcache_max) + 1);
}
bool
tcache_bins_ncached_max_write(tsd_t *tsd, char *settings, size_t len) {
assert(tcache_available(tsd));
tcache_t *tcache = tsd_tcachep_get(tsd);
assert(tcache != NULL);
cache_bin_info_t tcache_bin_info[TCACHE_NBINS_MAX];
tcache_bin_settings_backup(tcache, tcache_bin_info);
const char *bin_settings_segment_cur = settings;
size_t len_left = len;
assert(len_left != 0);
do {
size_t size_start, size_end;
size_t ncached_max;
bool err = multi_setting_parse_next(&bin_settings_segment_cur,
&len_left, &size_start, &size_end, &ncached_max);
if (err) {
return true;
}
if (size_end > TCACHE_MAXCLASS_LIMIT) {
size_end = TCACHE_MAXCLASS_LIMIT;
}
if (size_start > TCACHE_MAXCLASS_LIMIT ||
size_start > size_end) {
continue;
}
/* May get called before sz_init (during malloc_conf_init). */
szind_t bin_start = sz_size2index_compute(size_start);
szind_t bin_end = sz_size2index_compute(size_end);
if (ncached_max > CACHE_BIN_NCACHED_MAX) {
ncached_max = (size_t)CACHE_BIN_NCACHED_MAX;
}
for (szind_t i = bin_start; i <= bin_end; i++) {
cache_bin_info_init(&tcache_bin_info[i],
(cache_bin_sz_t)ncached_max);
}
} while (len_left > 0);
arena_t *assigned_arena = tcache->tcache_slow->arena;
tcache_cleanup(tsd);
tsd_tcache_data_init_with_bin_settings(tsd, assigned_arena,
tcache_bin_info);
return false;
}
static void
tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) {
tcache_slow_t *tcache_slow = tcache->tcache_slow;
@ -1180,7 +1247,7 @@ bool
tcache_boot(tsdn_t *tsdn, base_t *base) {
global_do_not_change_tcache_maxclass = sz_s2u(opt_tcache_max);
assert(global_do_not_change_tcache_maxclass <= TCACHE_MAXCLASS_LIMIT);
global_do_not_change_nbins =
global_do_not_change_tcache_nbins =
sz_size2index(global_do_not_change_tcache_maxclass) + 1;
if (malloc_mutex_init(&tcaches_mtx, "tcaches", WITNESS_RANK_TCACHES,

49
src/util.c Normal file
View file

@ -0,0 +1,49 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
#include "jemalloc/internal/util.h"
/* Reads the next size pair in a multi-sized option. */
bool
multi_setting_parse_next(const char **setting_segment_cur, size_t *len_left,
size_t *key_start, size_t *key_end, size_t *value) {
const char *cur = *setting_segment_cur;
char *end;
uintmax_t um;
set_errno(0);
/* First number, then '-' */
um = malloc_strtoumax(cur, &end, 0);
if (get_errno() != 0 || *end != '-') {
return true;
}
*key_start = (size_t)um;
cur = end + 1;
/* Second number, then ':' */
um = malloc_strtoumax(cur, &end, 0);
if (get_errno() != 0 || *end != ':') {
return true;
}
*key_end = (size_t)um;
cur = end + 1;
/* Last number */
um = malloc_strtoumax(cur, &end, 0);
if (get_errno() != 0) {
return true;
}
*value = (size_t)um;
/* Consume the separator if there is one. */
if (*end == '|') {
end++;
}
*len_left -= end - *setting_segment_cur;
*setting_segment_cur = end;
return false;
}