diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h new file mode 100644 index 00000000..dcac3b6a --- /dev/null +++ b/include/jemalloc/internal/base.h @@ -0,0 +1,103 @@ +#ifndef JEMALLOC_INTERNAL_BASE_H +#define JEMALLOC_INTERNAL_BASE_H + +#include "jemalloc/internal/edata.h" +#include "jemalloc/internal/ehooks.h" +#include "jemalloc/internal/mutex.h" + +enum metadata_thp_mode_e { + metadata_thp_disabled = 0, + /* + * Lazily enable hugepage for metadata. To avoid high RSS caused by THP + * + low usage arena (i.e. THP becomes a significant percentage), the + * "auto" option only starts using THP after a base allocator used up + * the first THP region. Starting from the second hugepage (in a single + * arena), "auto" behaves the same as "always", i.e. madvise hugepage + * right away. + */ + metadata_thp_auto = 1, + metadata_thp_always = 2, + metadata_thp_mode_limit = 3 +}; +typedef enum metadata_thp_mode_e metadata_thp_mode_t; + +#define METADATA_THP_DEFAULT metadata_thp_disabled +extern metadata_thp_mode_t opt_metadata_thp; +extern const char *metadata_thp_mode_names[]; + + +/* Embedded at the beginning of every block of base-managed virtual memory. */ +typedef struct base_block_s base_block_t; +struct base_block_s { + /* Total size of block's virtual memory mapping. */ + size_t size; + + /* Next block in list of base's blocks. */ + base_block_t *next; + + /* Tracks unused trailing space. */ + edata_t edata; +}; + +typedef struct base_s base_t; +struct base_s { + /* + * User-configurable extent hook functions. + */ + ehooks_t ehooks; + + /* Protects base_alloc() and base_stats_get() operations. */ + malloc_mutex_t mtx; + + /* Using THP when true (metadata_thp auto mode). */ + bool auto_thp_switched; + /* + * Most recent size class in the series of increasingly large base + * extents. Logarithmic spacing between subsequent allocations ensures + * that the total number of distinct mappings remains small. + */ + pszind_t pind_last; + + /* Serial number generation state. */ + size_t extent_sn_next; + + /* Chain of all blocks associated with base. */ + base_block_t *blocks; + + /* Heap of extents that track unused trailing space within blocks. */ + edata_heap_t avail[SC_NSIZES]; + + /* Stats, only maintained if config_stats. */ + size_t allocated; + size_t resident; + size_t mapped; + /* Number of THP regions touched. */ + size_t n_thp; +}; + +static inline unsigned +base_ind_get(const base_t *base) { + return ehooks_ind_get(&base->ehooks); +} + +static inline bool +metadata_thp_enabled(void) { + return (opt_metadata_thp != metadata_thp_disabled); +} + +base_t *b0get(void); +base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); +void base_delete(tsdn_t *tsdn, base_t *base); +ehooks_t *base_ehooks_get(base_t *base); +extent_hooks_t *base_extent_hooks_set(base_t *base, + extent_hooks_t *extent_hooks); +void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); +edata_t *base_alloc_edata(tsdn_t *tsdn, base_t *base); +void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, + size_t *resident, size_t *mapped, size_t *n_thp); +void base_prefork(tsdn_t *tsdn, base_t *base); +void base_postfork_parent(tsdn_t *tsdn, base_t *base); +void base_postfork_child(tsdn_t *tsdn, base_t *base); +bool base_boot(tsdn_t *tsdn); + +#endif /* JEMALLOC_INTERNAL_BASE_H */ diff --git a/include/jemalloc/internal/base_externs.h b/include/jemalloc/internal/base_externs.h deleted file mode 100644 index 2f241317..00000000 --- a/include/jemalloc/internal/base_externs.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_BASE_EXTERNS_H -#define JEMALLOC_INTERNAL_BASE_EXTERNS_H - -extern metadata_thp_mode_t opt_metadata_thp; -extern const char *metadata_thp_mode_names[]; - -base_t *b0get(void); -base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -void base_delete(tsdn_t *tsdn, base_t *base); -ehooks_t *base_ehooks_get(base_t *base); -extent_hooks_t *base_extent_hooks_set(base_t *base, - extent_hooks_t *extent_hooks); -void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); -edata_t *base_alloc_edata(tsdn_t *tsdn, base_t *base); -void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, - size_t *resident, size_t *mapped, size_t *n_thp); -void base_prefork(tsdn_t *tsdn, base_t *base); -void base_postfork_parent(tsdn_t *tsdn, base_t *base); -void base_postfork_child(tsdn_t *tsdn, base_t *base); -bool base_boot(tsdn_t *tsdn); - -#endif /* JEMALLOC_INTERNAL_BASE_EXTERNS_H */ diff --git a/include/jemalloc/internal/base_inlines.h b/include/jemalloc/internal/base_inlines.h deleted file mode 100644 index 221fca81..00000000 --- a/include/jemalloc/internal/base_inlines.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_BASE_INLINES_H -#define JEMALLOC_INTERNAL_BASE_INLINES_H - -static inline unsigned -base_ind_get(const base_t *base) { - return ehooks_ind_get(&base->ehooks); -} - -static inline bool -metadata_thp_enabled(void) { - return (opt_metadata_thp != metadata_thp_disabled); -} -#endif /* JEMALLOC_INTERNAL_BASE_INLINES_H */ diff --git a/include/jemalloc/internal/base_types.h b/include/jemalloc/internal/base_types.h deleted file mode 100644 index b6db77df..00000000 --- a/include/jemalloc/internal/base_types.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_BASE_TYPES_H -#define JEMALLOC_INTERNAL_BASE_TYPES_H - -typedef struct base_block_s base_block_t; -typedef struct base_s base_t; - -#define METADATA_THP_DEFAULT metadata_thp_disabled - -/* - * In auto mode, arenas switch to huge pages for the base allocator on the - * second base block. a0 switches to thp on the 5th block (after 20 megabytes - * of metadata), since more metadata (e.g. rtree nodes) come from a0's base. - */ - -#define BASE_AUTO_THP_THRESHOLD 2 -#define BASE_AUTO_THP_THRESHOLD_A0 5 - -typedef enum { - metadata_thp_disabled = 0, - /* - * Lazily enable hugepage for metadata. To avoid high RSS caused by THP - * + low usage arena (i.e. THP becomes a significant percentage), the - * "auto" option only starts using THP after a base allocator used up - * the first THP region. Starting from the second hugepage (in a single - * arena), "auto" behaves the same as "always", i.e. madvise hugepage - * right away. - */ - metadata_thp_auto = 1, - metadata_thp_always = 2, - metadata_thp_mode_limit = 3 -} metadata_thp_mode_t; - -#endif /* JEMALLOC_INTERNAL_BASE_TYPES_H */ diff --git a/include/jemalloc/internal/edata_cache.h b/include/jemalloc/internal/edata_cache.h index 9cb0d1c8..73ac7af8 100644 --- a/include/jemalloc/internal/edata_cache.h +++ b/include/jemalloc/internal/edata_cache.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_EDATA_CACHE_H #define JEMALLOC_INTERNAL_EDATA_CACHE_H +#include "jemalloc/internal/base.h" + /* * A cache of edata_t structures allocated via base_alloc_edata (as opposed to * the underlying extents they describe). The contents of returned edata_t diff --git a/include/jemalloc/internal/jemalloc_internal_includes.h b/include/jemalloc/internal/jemalloc_internal_includes.h index 75a94d3a..72b5a72a 100644 --- a/include/jemalloc/internal/jemalloc_internal_includes.h +++ b/include/jemalloc/internal/jemalloc_internal_includes.h @@ -40,7 +40,6 @@ /* TYPES */ /******************************************************************************/ -#include "jemalloc/internal/base_types.h" #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/tcache_types.h" #include "jemalloc/internal/prof_types.h" @@ -51,7 +50,6 @@ #include "jemalloc/internal/prof_structs.h" #include "jemalloc/internal/arena_structs.h" -#include "jemalloc/internal/base_structs.h" #include "jemalloc/internal/tcache_structs.h" #include "jemalloc/internal/background_thread_structs.h" @@ -60,7 +58,6 @@ /******************************************************************************/ #include "jemalloc/internal/jemalloc_internal_externs.h" -#include "jemalloc/internal/base_externs.h" #include "jemalloc/internal/arena_externs.h" #include "jemalloc/internal/large_externs.h" #include "jemalloc/internal/tcache_externs.h" @@ -72,7 +69,6 @@ /******************************************************************************/ #include "jemalloc/internal/jemalloc_internal_inlines_a.h" -#include "jemalloc/internal/base_inlines.h" /* * Include portions of arena code interleaved with tcache code in order to * resolve circular dependencies. diff --git a/src/base.c b/src/base.c index c006774e..595b7710 100644 --- a/src/base.c +++ b/src/base.c @@ -7,6 +7,15 @@ #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/sz.h" +/* + * In auto mode, arenas switch to huge pages for the base allocator on the + * second base block. a0 switches to thp on the 5th block (after 20 megabytes + * of metadata), since more metadata (e.g. rtree nodes) come from a0's base. + */ + +#define BASE_AUTO_THP_THRESHOLD 2 +#define BASE_AUTO_THP_THRESHOLD_A0 5 + /******************************************************************************/ /* Data. */