diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h index 005f422e..de1c8149 100644 --- a/include/jemalloc/internal/arena_inlines_b.h +++ b/include/jemalloc/internal/arena_inlines_b.h @@ -9,7 +9,7 @@ #include "jemalloc/internal/emap.h" #include "jemalloc/internal/jemalloc_internal_inlines_b.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/large_externs.h" +#include "jemalloc/internal/large.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/prof_externs.h" #include "jemalloc/internal/prof_structs.h" diff --git a/include/jemalloc/internal/background_thread_structs.h b/include/jemalloc/internal/background_thread.h similarity index 58% rename from include/jemalloc/internal/background_thread_structs.h rename to include/jemalloc/internal/background_thread.h index d56673da..db26f1b8 100644 --- a/include/jemalloc/internal/background_thread_structs.h +++ b/include/jemalloc/internal/background_thread.h @@ -1,11 +1,10 @@ -#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H -#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H +#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_H +#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_H #include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/base.h" #include "jemalloc/internal/mutex.h" -/* This file really combines "structs" and "types", but only transitionally. */ - #if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK) # define JEMALLOC_PTHREAD_CREATE_WRAPPER #endif @@ -66,4 +65,33 @@ struct background_thread_stats_s { }; typedef struct background_thread_stats_s background_thread_stats_t; -#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_STRUCTS_H */ +extern bool opt_background_thread; +extern size_t opt_max_background_threads; +extern malloc_mutex_t background_thread_lock; +extern atomic_b_t background_thread_enabled_state; +extern size_t n_background_threads; +extern size_t max_background_threads; +extern background_thread_info_t *background_thread_info; + +bool background_thread_create(tsd_t *tsd, unsigned arena_ind); +bool background_threads_enable(tsd_t *tsd); +bool background_threads_disable(tsd_t *tsd); +bool background_thread_is_started(background_thread_info_t *info); +void background_thread_wakeup_early( + background_thread_info_t *info, nstime_t *remaining_sleep); +void background_thread_prefork0(tsdn_t *tsdn); +void background_thread_prefork1(tsdn_t *tsdn); +void background_thread_postfork_parent(tsdn_t *tsdn); +void background_thread_postfork_child(tsdn_t *tsdn); +bool background_thread_stats_read( + tsdn_t *tsdn, background_thread_stats_t *stats); +void background_thread_ctl_init(tsdn_t *tsdn); + +#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER +extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *, + void *(*)(void *), void *__restrict); +#endif +bool background_thread_boot0(void); +bool background_thread_boot1(tsdn_t *tsdn, base_t *base); + +#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_H */ diff --git a/include/jemalloc/internal/background_thread_externs.h b/include/jemalloc/internal/background_thread_externs.h deleted file mode 100644 index efc0aaa4..00000000 --- a/include/jemalloc/internal/background_thread_externs.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H -#define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H - -#include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/background_thread_structs.h" -#include "jemalloc/internal/base.h" -#include "jemalloc/internal/mutex.h" - -extern bool opt_background_thread; -extern size_t opt_max_background_threads; -extern malloc_mutex_t background_thread_lock; -extern atomic_b_t background_thread_enabled_state; -extern size_t n_background_threads; -extern size_t max_background_threads; -extern background_thread_info_t *background_thread_info; - -bool background_thread_create(tsd_t *tsd, unsigned arena_ind); -bool background_threads_enable(tsd_t *tsd); -bool background_threads_disable(tsd_t *tsd); -bool background_thread_is_started(background_thread_info_t *info); -void background_thread_wakeup_early( - background_thread_info_t *info, nstime_t *remaining_sleep); -void background_thread_prefork0(tsdn_t *tsdn); -void background_thread_prefork1(tsdn_t *tsdn); -void background_thread_postfork_parent(tsdn_t *tsdn); -void background_thread_postfork_child(tsdn_t *tsdn); -bool background_thread_stats_read( - tsdn_t *tsdn, background_thread_stats_t *stats); -void background_thread_ctl_init(tsdn_t *tsdn); - -#ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER -extern int pthread_create_wrapper(pthread_t *__restrict, const pthread_attr_t *, - void *(*)(void *), void *__restrict); -#endif -bool background_thread_boot0(void); -bool background_thread_boot1(tsdn_t *tsdn, base_t *base); - -#endif /* JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H */ diff --git a/include/jemalloc/internal/background_thread_inlines.h b/include/jemalloc/internal/background_thread_inlines.h index e822a3f7..67517626 100644 --- a/include/jemalloc/internal/background_thread_inlines.h +++ b/include/jemalloc/internal/background_thread_inlines.h @@ -4,7 +4,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/arena_inlines_a.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/background_thread_externs.h" +#include "jemalloc/internal/background_thread.h" JEMALLOC_ALWAYS_INLINE bool background_thread_enabled(void) { diff --git a/include/jemalloc/internal/bin.h b/include/jemalloc/internal/bin.h index 45c6c5dd..90fc3d31 100644 --- a/include/jemalloc/internal/bin.h +++ b/include/jemalloc/internal/bin.h @@ -4,11 +4,17 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/bin_info.h" #include "jemalloc/internal/bin_stats.h" -#include "jemalloc/internal/bin_types.h" +#include "jemalloc/internal/div.h" #include "jemalloc/internal/edata.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/sc.h" +/* Forward decl; only used as a pointer type in bin_choose. */ +typedef struct arena_s arena_t; + +#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH) +#define N_BIN_SHARDS_DEFAULT 1 + /* * A bin contains a set of extents that are currently being used for slab * allocations. @@ -102,30 +108,11 @@ void *bin_current_slab_addr(tsdn_t *tsdn, bin_t *bin); bin_t *bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned *binshard_p); -/* Stats. */ -static inline void -bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) { - malloc_mutex_lock(tsdn, &bin->lock); - bin->stats.nrequests += n; - malloc_mutex_unlock(tsdn, &bin->lock); -} - -static inline void -bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) { - malloc_mutex_lock(tsdn, &bin->lock); - malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock); - bin_stats_t *stats = &dst_bin_stats->stats_data; - stats->nmalloc += bin->stats.nmalloc; - stats->ndalloc += bin->stats.ndalloc; - stats->nrequests += bin->stats.nrequests; - stats->curregs += bin->stats.curregs; - stats->nfills += bin->stats.nfills; - stats->nflushes += bin->stats.nflushes; - stats->nslabs += bin->stats.nslabs; - stats->reslabs += bin->stats.reslabs; - stats->curslabs += bin->stats.curslabs; - stats->nonfull_slabs += bin->stats.nonfull_slabs; - malloc_mutex_unlock(tsdn, &bin->lock); -} +typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t; +struct bin_dalloc_locked_info_s { + div_info_t div_info; + uint32_t nregs; + uint64_t ndalloc; +}; #endif /* JEMALLOC_INTERNAL_BIN_H */ diff --git a/include/jemalloc/internal/bin_inlines.h b/include/jemalloc/internal/bin_inlines.h index 31fe4818..5d7a9d04 100644 --- a/include/jemalloc/internal/bin_inlines.h +++ b/include/jemalloc/internal/bin_inlines.h @@ -7,20 +7,33 @@ #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/div.h" #include "jemalloc/internal/edata.h" -#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/mutex.h" -/* - * The dalloc bin info contains just the information that the common paths need - * during tcache flushes. By force-inlining these paths, and using local copies - * of data (so that the compiler knows it's constant), we avoid a whole bunch of - * redundant loads and stores by leaving this information in registers. - */ -typedef struct bin_dalloc_locked_info_s bin_dalloc_locked_info_t; -struct bin_dalloc_locked_info_s { - div_info_t div_info; - uint32_t nregs; - uint64_t ndalloc; -}; +/* Stats. */ +static inline void +bin_stats_nrequests_add(tsdn_t *tsdn, bin_t *bin, uint64_t n) { + malloc_mutex_lock(tsdn, &bin->lock); + bin->stats.nrequests += n; + malloc_mutex_unlock(tsdn, &bin->lock); +} + +static inline void +bin_stats_merge(tsdn_t *tsdn, bin_stats_data_t *dst_bin_stats, bin_t *bin) { + malloc_mutex_lock(tsdn, &bin->lock); + malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock); + bin_stats_t *stats = &dst_bin_stats->stats_data; + stats->nmalloc += bin->stats.nmalloc; + stats->ndalloc += bin->stats.ndalloc; + stats->nrequests += bin->stats.nrequests; + stats->curregs += bin->stats.curregs; + stats->nfills += bin->stats.nfills; + stats->nflushes += bin->stats.nflushes; + stats->nslabs += bin->stats.nslabs; + stats->reslabs += bin->stats.reslabs; + stats->curslabs += bin->stats.curslabs; + stats->nonfull_slabs += bin->stats.nonfull_slabs; + malloc_mutex_unlock(tsdn, &bin->lock); +} /* Find the region index of a pointer within a slab. */ JEMALLOC_ALWAYS_INLINE size_t @@ -52,14 +65,6 @@ bin_slab_regind(const bin_dalloc_locked_info_t *info, szind_t binind, return regind; } -JEMALLOC_ALWAYS_INLINE void -bin_dalloc_locked_begin( - bin_dalloc_locked_info_t *info, szind_t binind) { - info->div_info = arena_binind_div_info[binind]; - info->nregs = bin_infos[binind].nregs; - info->ndalloc = 0; -} - /* * Does the deallocation work associated with freeing a single pointer (a * "step") in between a bin_dalloc_locked begin and end call. diff --git a/include/jemalloc/internal/bin_types.h b/include/jemalloc/internal/bin_types.h deleted file mode 100644 index b6bad37e..00000000 --- a/include/jemalloc/internal/bin_types.h +++ /dev/null @@ -1,21 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H -#define JEMALLOC_INTERNAL_BIN_TYPES_H - -#include "jemalloc/internal/jemalloc_preamble.h" -#include "jemalloc/internal/sc.h" - -#define BIN_SHARDS_MAX (1 << EDATA_BITS_BINSHARD_WIDTH) -#define N_BIN_SHARDS_DEFAULT 1 - -/* Used in TSD static initializer only. Real init in arena_bind(). */ -#define TSD_BINSHARDS_ZERO_INITIALIZER \ - { \ - { UINT8_MAX } \ - } - -typedef struct tsd_binshards_s tsd_binshards_t; -struct tsd_binshards_s { - uint8_t binshard[SC_NBINS]; -}; - -#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */ diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index e7a8221c..09f1825c 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/arena_stats.h" -#include "jemalloc/internal/background_thread_structs.h" +#include "jemalloc/internal/background_thread.h" #include "jemalloc/internal/bin_stats.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/malloc_io.h" diff --git a/include/jemalloc/internal/jemalloc_internal_includes.h b/include/jemalloc/internal/jemalloc_internal_includes.h index d198d09c..63b65599 100644 --- a/include/jemalloc/internal/jemalloc_internal_includes.h +++ b/include/jemalloc/internal/jemalloc_internal_includes.h @@ -51,7 +51,6 @@ #include "jemalloc/internal/prof_structs.h" #include "jemalloc/internal/arena_structs.h" #include "jemalloc/internal/tcache_structs.h" -#include "jemalloc/internal/background_thread_structs.h" /******************************************************************************/ /* EXTERNS */ @@ -59,11 +58,11 @@ #include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/arena_externs.h" -#include "jemalloc/internal/large_externs.h" +#include "jemalloc/internal/large.h" #include "jemalloc/internal/tcache_externs.h" #include "jemalloc/internal/malloc_dispatch.h" #include "jemalloc/internal/prof_externs.h" -#include "jemalloc/internal/background_thread_externs.h" +#include "jemalloc/internal/background_thread.h" /******************************************************************************/ /* INLINES */ diff --git a/include/jemalloc/internal/large_externs.h b/include/jemalloc/internal/large.h similarity index 87% rename from include/jemalloc/internal/large_externs.h rename to include/jemalloc/internal/large.h index 82abd7de..e28841c1 100644 --- a/include/jemalloc/internal/large_externs.h +++ b/include/jemalloc/internal/large.h @@ -1,5 +1,5 @@ -#ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H -#define JEMALLOC_INTERNAL_LARGE_EXTERNS_H +#ifndef JEMALLOC_INTERNAL_LARGE_H +#define JEMALLOC_INTERNAL_LARGE_H #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/edata.h" @@ -20,4 +20,4 @@ void large_prof_info_get( void large_prof_tctx_reset(edata_t *edata); void large_prof_info_set(edata_t *edata, prof_tctx_t *tctx, size_t size); -#endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */ +#endif /* JEMALLOC_INTERNAL_LARGE_H */ diff --git a/include/jemalloc/internal/malloc_dispatch_inlines.h b/include/jemalloc/internal/malloc_dispatch_inlines.h index e18192bf..5632e27a 100644 --- a/include/jemalloc/internal/malloc_dispatch_inlines.h +++ b/include/jemalloc/internal/malloc_dispatch_inlines.h @@ -9,7 +9,7 @@ #include "jemalloc/internal/emap.h" #include "jemalloc/internal/jemalloc_internal_inlines_b.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/large_externs.h" +#include "jemalloc/internal/large.h" #include "jemalloc/internal/malloc_dispatch.h" #include "jemalloc/internal/safety_check.h" #include "jemalloc/internal/sc.h" diff --git a/include/jemalloc/internal/tcache_inlines.h b/include/jemalloc/internal/tcache_inlines.h index b3ce81bd..181db1b3 100644 --- a/include/jemalloc/internal/tcache_inlines.h +++ b/include/jemalloc/internal/tcache_inlines.h @@ -6,7 +6,7 @@ #include "jemalloc/internal/bin.h" #include "jemalloc/internal/jemalloc_internal_inlines_b.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/large_externs.h" +#include "jemalloc/internal/large.h" #include "jemalloc/internal/san.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" diff --git a/include/jemalloc/internal/tsd_binshards.h b/include/jemalloc/internal/tsd_binshards.h new file mode 100644 index 00000000..a4afc496 --- /dev/null +++ b/include/jemalloc/internal/tsd_binshards.h @@ -0,0 +1,24 @@ +#ifndef JEMALLOC_INTERNAL_TSD_BINSHARDS_H +#define JEMALLOC_INTERNAL_TSD_BINSHARDS_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/sc.h" + +/* + * Per-thread cache of bin-shard assignments. This lives in its own header + * (separate from bin.h) so that tsd_internals.h can pull it in for X-macro + * expansion without dragging in mutex.h, which itself depends on TSD machinery + * and would form an include-order dependency cycle. + */ + +#define TSD_BINSHARDS_ZERO_INITIALIZER \ + { \ + { UINT8_MAX } \ + } + +typedef struct tsd_binshards_s tsd_binshards_t; +struct tsd_binshards_s { + uint8_t binshard[SC_NBINS]; +}; + +#endif /* JEMALLOC_INTERNAL_TSD_BINSHARDS_H */ diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index 46b4930f..a7b6fa5e 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -6,7 +6,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/assert.h" -#include "jemalloc/internal/bin_types.h" +#include "jemalloc/internal/tsd_binshards.h" #include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/peak.h" #include "jemalloc/internal/prof_types.h" diff --git a/src/arena.c b/src/arena.c index 3f4e7b65..e8d90458 100644 --- a/src/arena.c +++ b/src/arena.c @@ -36,6 +36,14 @@ static pa_central_t arena_pa_central_global; div_info_t arena_binind_div_info[SC_NBINS]; +JET_EXTERN void +bin_dalloc_locked_begin( + bin_dalloc_locked_info_t *info, szind_t binind) { + info->div_info = arena_binind_div_info[binind]; + info->nregs = bin_infos[binind].nregs; + info->ndalloc = 0; +} + size_t opt_oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; size_t oversize_threshold = OVERSIZE_THRESHOLD_DEFAULT; diff --git a/test/unit/bin.c b/test/unit/bin.c index 08dd4665..a7091544 100644 --- a/test/unit/bin.c +++ b/test/unit/bin.c @@ -1,5 +1,8 @@ #include "test/jemalloc_test.h" +extern void bin_dalloc_locked_begin( + bin_dalloc_locked_info_t *info, szind_t binind); + #define INVALID_ARENA_IND ((1U << MALLOCX_ARENA_BITS) - 1) /* Create a page-aligned mock slab with all regions free. */ diff --git a/test/unit/slab.c b/test/unit/slab.c index d98663e8..e7adafea 100644 --- a/test/unit/slab.c +++ b/test/unit/slab.c @@ -1,5 +1,8 @@ #include "test/jemalloc_test.h" +extern void bin_dalloc_locked_begin( + bin_dalloc_locked_info_t *info, szind_t binind); + #define INVALID_ARENA_IND ((1U << MALLOCX_ARENA_BITS) - 1) TEST_BEGIN(test_bin_slab_regind) {