From 3f9f2833f6228e07673d75c9bce6f5fb58c5f3b0 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Tue, 20 Nov 2018 13:51:32 -0800 Subject: [PATCH] Add opt.bin_shards to specify number of bin shards. The option uses the same format as "slab_sizes" to specify number of shards for each bin size. --- include/jemalloc/internal/bin.h | 10 ++--- include/jemalloc/internal/extent_structs.h | 4 -- src/arena.c | 11 ++--- src/bin.c | 48 +++++++++++++++++----- src/jemalloc.c | 39 ++++++++++++++---- 5 files changed, 81 insertions(+), 31 deletions(-) diff --git a/include/jemalloc/internal/bin.h b/include/jemalloc/internal/bin.h index 3fddef73..baa0acf3 100644 --- a/include/jemalloc/internal/bin.h +++ b/include/jemalloc/internal/bin.h @@ -8,9 +8,7 @@ #include "jemalloc/internal/sc.h" #define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH) - -extern unsigned opt_bin_shard_maxszind; -extern unsigned opt_n_bin_shards; +#define N_BIN_SHARDS_DEFAULT 1 /* * A bin contains a set of extents that are currently being used for slab @@ -93,8 +91,10 @@ struct bins_s { bin_t *bin_shards; }; -void bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]); -void bin_boot(); +void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]); +bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards); +void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]); /* Initializes a bin to empty. Returns true on error. */ bool bin_init(bin_t *bin); diff --git a/include/jemalloc/internal/extent_structs.h b/include/jemalloc/internal/extent_structs.h index 16264528..ceb18979 100644 --- a/include/jemalloc/internal/extent_structs.h +++ b/include/jemalloc/internal/extent_structs.h @@ -128,10 +128,6 @@ struct extent_s { #define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) #define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT) -/* Will make dynamic options. */ -#define OPT_N_BIN_SHARDS (1) -#define OPT_BIN_SHARD_MAXSZIND (0) - #define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT) #define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) diff --git a/src/arena.c b/src/arena.c index 893c9b56..7017bd7a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1346,8 +1346,7 @@ bin_t * arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned *binshard) { bin_t *bin; - if (binind >= opt_bin_shard_maxszind || tsdn_null(tsdn) || - tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { + if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { *binshard = 0; } else { *binshard = tsd_binshard_get(tsdn_tsd(tsdn)) % @@ -1923,9 +1922,11 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } } - size_t arena_size = sizeof(arena_t) + - sizeof(bin_t) * opt_n_bin_shards * opt_bin_shard_maxszind + - sizeof(bin_t) * (SC_NBINS - opt_bin_shard_maxszind); + unsigned nbins_total = 0; + for (i = 0; i < SC_NBINS; i++) { + nbins_total += bin_infos[i].n_shards; + } + size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total; arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); if (arena == NULL) { goto label_error; diff --git a/src/bin.c b/src/bin.c index 8dd964fa..bca6b12c 100644 --- a/src/bin.c +++ b/src/bin.c @@ -6,13 +6,11 @@ #include "jemalloc/internal/sc.h" #include "jemalloc/internal/witness.h" -unsigned opt_bin_shard_maxszind; -unsigned opt_n_bin_shards; - bin_info_t bin_infos[SC_NBINS]; -void -bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]) { +static void +bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], + bin_info_t bin_infos[SC_NBINS]) { for (unsigned i = 0; i < SC_NBINS; i++) { bin_info_t *bin_info = &bin_infos[i]; sc_t *sc = &sc_data->sc[i]; @@ -21,19 +19,49 @@ bin_infos_init(sc_data_t *sc_data, bin_info_t bin_infos[SC_NBINS]) { bin_info->slab_size = (sc->pgs << LG_PAGE); bin_info->nregs = (uint32_t)(bin_info->slab_size / bin_info->reg_size); - bin_info->n_shards = (i < opt_bin_shard_maxszind) ? opt_n_bin_shards : 1; + bin_info->n_shards = bin_shard_sizes[i]; bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER( bin_info->nregs); bin_info->bitmap_info = bitmap_info; } } +bool +bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards) { + if (nshards > BIN_SHARDS_MAX || nshards == 0) { + return true; + } + + if (start_size > SC_SMALL_MAXCLASS) { + return false; + } + if (end_size > SC_SMALL_MAXCLASS) { + end_size = SC_SMALL_MAXCLASS; + } + + /* Compute the index since this may happen before sz init. */ + szind_t ind1 = sz_size2index_compute(start_size); + szind_t ind2 = sz_size2index_compute(end_size); + for (unsigned i = ind1; i <= ind2; i++) { + bin_shard_sizes[i] = (unsigned)nshards; + } + + return false; +} + void -bin_boot(sc_data_t *sc_data) { +bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) { + /* Load the default number of shards. */ + for (unsigned i = 0; i < SC_NBINS; i++) { + bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT; + } +} + +void +bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { assert(sc_data->initialized); - opt_bin_shard_maxszind = OPT_BIN_SHARD_MAXSZIND; - opt_n_bin_shards = OPT_N_BIN_SHARDS; - bin_infos_init(sc_data, bin_infos); + bin_infos_init(sc_data, bin_shard_sizes, bin_infos); } bool diff --git a/src/jemalloc.c b/src/jemalloc.c index c635ecb4..1f7ed2eb 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -767,9 +767,10 @@ init_opt_stats_print_opts(const char *v, size_t vlen) { assert(opts_len == strlen(opt_stats_print_opts)); } +/* Reads the next size pair in a multi-sized option. */ static bool -malloc_conf_slab_sizes_next(const char **slab_size_segment_cur, - size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *pgs) { +malloc_conf_multi_sizes_next(const char **slab_size_segment_cur, + size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) { const char *cur = *slab_size_segment_cur; char *end; uintmax_t um; @@ -797,7 +798,7 @@ malloc_conf_slab_sizes_next(const char **slab_size_segment_cur, if (get_errno() != 0) { return true; } - *pgs = (size_t)um; + *new_size = (size_t)um; /* Consume the separator if there is one. */ if (*end == '|') { @@ -923,7 +924,7 @@ malloc_slow_flag_init(void) { } static void -malloc_conf_init(sc_data_t *sc_data) { +malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { unsigned i; char buf[PATH_MAX + 1]; const char *opts, *k, *v; @@ -1161,6 +1162,28 @@ malloc_conf_init(sc_data_t *sc_data) { } CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, UINT_MAX, yes, no, false) + if (CONF_MATCH("bin_shards")) { + const char *bin_shards_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t size_start; + size_t size_end; + size_t nshards; + bool err = malloc_conf_multi_sizes_next( + &bin_shards_segment_cur, &vlen_left, + &size_start, &size_end, &nshards); + if (err || bin_update_shard_size( + bin_shard_sizes, size_start, + size_end, nshards)) { + malloc_conf_error( + "Invalid settings for " + "bin_shards", k, klen, v, + vlen); + break; + } + } while (vlen_left > 0); + continue; + } CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : @@ -1256,7 +1279,7 @@ malloc_conf_init(sc_data_t *sc_data) { size_t slab_start; size_t slab_end; size_t pgs; - err = malloc_conf_slab_sizes_next( + err = malloc_conf_multi_sizes_next( &slab_size_segment_cur, &vlen_left, &slab_start, &slab_end, &pgs); @@ -1390,6 +1413,8 @@ malloc_init_hard_a0_locked() { * out of sc_data_global are final. */ sc_boot(&sc_data); + unsigned bin_shard_sizes[SC_NBINS]; + bin_shard_sizes_boot(bin_shard_sizes); /* * prof_boot0 only initializes opt_prof_prefix. We need to do it before * we parse malloc_conf options, in case malloc_conf parsing overwrites @@ -1398,9 +1423,9 @@ malloc_init_hard_a0_locked() { if (config_prof) { prof_boot0(); } - malloc_conf_init(&sc_data); + malloc_conf_init(&sc_data, bin_shard_sizes); sz_boot(&sc_data); - bin_boot(&sc_data); + bin_boot(&sc_data, bin_shard_sizes); if (opt_stats_print) { /* Print statistics at exit. */