romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-04-14 22:51:50 +03:00
Code Issues Projects Releases Packages Wiki Activity

[SEC] Make SEC owned by hpa_shard, simplify the code, add stats, lock per bin

Browse source
This commit is contained in:
Slobodan Predolac 2025-10-30 16:05:04 -07:00 • committed by guangli-dai
parent d930391cf3
commit b5da68dbc3
35 changed files with 1264 additions and 1257 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Makefile.in
View file

@ -135,7 +135,6 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/nstime.c \
$(srcroot)src/pa.c \
$(srcroot)src/pa_extra.c \
$(srcroot)src/pai.c \
$(srcroot)src/pac.c \
$(srcroot)src/pages.c \
$(srcroot)src/peak_event.c \
@ -230,6 +229,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hook.c \
$(srcroot)test/unit/hpa.c \
$(srcroot)test/unit/hpa_sec_integration.c \
$(srcroot)test/unit/hpa_thp_always.c \
$(srcroot)test/unit/hpa_vectorized_madvise.c \
$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \

2
include/jemalloc/internal/arena_externs.h
View file

@ -46,7 +46,7 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
hpa_shard_stats_t *hpastats, sec_stats_t *secstats);
hpa_shard_stats_t *hpastats);
void arena_handle_deferred_work(tsdn_t *tsdn, arena_t *arena);
edata_t *arena_extent_alloc_large(
tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero);

1
include/jemalloc/internal/ctl.h
View file

@ -51,7 +51,6 @@ typedef struct ctl_arena_stats_s {
arena_stats_large_t lstats[SC_NSIZES - SC_NBINS];
pac_estats_t estats[SC_NPSIZES];
hpa_shard_stats_t hpastats;
sec_stats_t secstats;
} ctl_arena_stats_t;
typedef struct ctl_stats_s {

19
include/jemalloc/internal/hpa.h
View file

@ -12,6 +12,7 @@
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/pai.h"
#include "jemalloc/internal/psset.h"
#include "jemalloc/internal/sec.h"
typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
struct hpa_shard_nonderived_stats_s {
@ -57,6 +58,7 @@ typedef struct hpa_shard_stats_s hpa_shard_stats_t;
struct hpa_shard_stats_s {
psset_stats_t psset_stats;
hpa_shard_nonderived_stats_t nonderived_stats;
sec_stats_t secstats;
};
typedef struct hpa_shard_s hpa_shard_t;
@ -69,14 +71,17 @@ struct hpa_shard_s {
/* The central allocator we get our hugepages from. */
hpa_central_t *central;
/* Protects most of this shard's state. */
malloc_mutex_t mtx;
/*
* Guards the shard's access to the central allocator (preventing
* multiple threads operating on this shard from accessing the central
* allocator).
*/
malloc_mutex_t grow_mtx;
/* The base metadata allocator. */
base_t *base;
@ -87,6 +92,9 @@ struct hpa_shard_s {
*/
edata_cache_fast_t ecf;
/* Small extent cache (not guarded by mtx) */
JEMALLOC_ALIGNED(CACHELINE) sec_t sec;
psset_t psset;
/*
@ -142,9 +150,9 @@ bool hpa_hugepage_size_exceeds_limit(void);
* just that it can function properly given the system it's running on.
*/
bool hpa_supported(void);
bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts);
bool hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts);
void hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src);
void hpa_shard_stats_merge(
@ -157,6 +165,8 @@ void hpa_shard_stats_merge(
*/
void hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard);
/* Flush caches that shard may be using */
void hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_set_deferral_allowed(
tsdn_t *tsdn, hpa_shard_t *shard, bool deferral_allowed);
@ -164,8 +174,9 @@ void hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard);
/*
* We share the fork ordering with the PA and arena prefork handling; that's why
* these are 3 and 4 rather than 0 and 1.
* these are 2, 3 and 4 rather than 0 and 1.
*/
void hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_prefork4(tsdn_t *tsdn, hpa_shard_t *shard);
void hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard);

12
include/jemalloc/internal/pa.h
View file

@ -96,12 +96,6 @@ struct pa_shard_s {
/* Allocates from a PAC. */
pac_t pac;
/*
* We place a small extent cache in front of the HPA, since we intend
* these configurations to use many fewer arenas, and therefore have a
* higher risk of hot locks.
*/
sec_t hpa_sec;
hpa_shard_t hpa_shard;
/* The source of edata_t objects. */
@ -166,6 +160,9 @@ void pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard);
*/
void pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard);
/* Flush any caches used by shard */
void pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard);
/* Gets an edata for the given allocation. */
edata_t *pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size,
size_t alignment, bool slab, szind_t szind, bool zero, bool guarded,
@ -233,8 +230,7 @@ void pa_shard_basic_stats_merge(
void pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
size_t *resident);
hpa_shard_stats_t *hpa_stats_out, size_t *resident);
/*
* Reads the PA-owned mutex stats into the output stats array, at the

37
include/jemalloc/internal/pai.h
View file

@ -13,15 +13,6 @@ struct pai_s {
edata_t *(*alloc)(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
/*
* Returns the number of extents added to the list (which may be fewer
* than requested, in case of OOM). The list should already be
* initialized. The only alignment guarantee is page-alignment, and
* the results are not necessarily zeroed.
*/
size_t (*alloc_batch)(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated);
bool (*expand)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero,
bool *deferred_work_generated);
@ -29,9 +20,6 @@ struct pai_s {
size_t old_size, size_t new_size, bool *deferred_work_generated);
void (*dalloc)(tsdn_t *tsdn, pai_t *self, edata_t *edata,
bool *deferred_work_generated);
/* This function empties out list as a side-effect of being called. */
void (*dalloc_batch)(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated);
uint64_t (*time_until_deferred_work)(tsdn_t *tsdn, pai_t *self);
};
@ -47,14 +35,6 @@ pai_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
frequent_reuse, deferred_work_generated);
}
static inline size_t
pai_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated) {
return self->alloc_batch(tsdn, self, size, nallocs, results,
frequent_reuse, deferred_work_generated);
}
static inline bool
pai_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated) {
@ -75,26 +55,9 @@ pai_dalloc(
self->dalloc(tsdn, self, edata, deferred_work_generated);
}
static inline void
pai_dalloc_batch(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
bool *deferred_work_generated) {
self->dalloc_batch(tsdn, self, list, deferred_work_generated);
}
static inline uint64_t
pai_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
return self->time_until_deferred_work(tsdn, self);
}
/*
* An implementation of batch allocation that simply calls alloc once for
* each item in the list.
*/
size_t pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated);
/* Ditto, for dalloc. */
void pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated);
#endif /* JEMALLOC_INTERNAL_PAI_H */

129
include/jemalloc/internal/sec.h
View file

@ -17,91 +17,104 @@
* knowledge of the underlying PAI implementation).
*/
/*
* For now, this is just one field; eventually, we'll probably want to get more
* fine-grained data out (like per-size class statistics).
*/
typedef struct sec_bin_stats_s sec_bin_stats_t;
struct sec_bin_stats_s {
/* Number of alloc requests that did not find extent in this bin */
size_t nmisses;
/* Number of successful alloc requests. */
size_t nhits;
/* Number of dallocs causing the flush */
size_t ndalloc_flush;
/* Number of dallocs not causing the flush */
size_t ndalloc_noflush;
/* Number of fills that hit max_bytes */
size_t noverfills;
};
typedef struct sec_stats_s sec_stats_t;
struct sec_stats_s {
/* Sum of bytes_cur across all shards. */
size_t bytes;
/* Totals of bin_stats. */
sec_bin_stats_t total;
};
static inline void
sec_bin_stats_init(sec_bin_stats_t *stats) {
stats->ndalloc_flush = 0;
stats->nmisses = 0;
stats->nhits = 0;
stats->ndalloc_noflush = 0;
stats->noverfills = 0;
}
static inline void
sec_bin_stats_accum(sec_bin_stats_t *dst, sec_bin_stats_t *src) {
dst->nmisses += src->nmisses;
dst->nhits += src->nhits;
dst->ndalloc_flush += src->ndalloc_flush;
dst->ndalloc_noflush += src->ndalloc_noflush;
dst->noverfills += src->noverfills;
}
static inline void
sec_stats_accum(sec_stats_t *dst, sec_stats_t *src) {
dst->bytes += src->bytes;
sec_bin_stats_accum(&dst->total, &src->total);
}
/* A collections of free extents, all of the same size. */
typedef struct sec_bin_s sec_bin_t;
struct sec_bin_s {
/*
* When we fail to fulfill an allocation, we do a batch-alloc on the
* underlying allocator to fill extra items, as well. We drop the SEC
* lock while doing so, to allow operations on other bins to succeed.
* That introduces the possibility of other threads also trying to
* allocate out of this bin, failing, and also going to the backing
* allocator. To avoid a thundering herd problem in which lots of
* threads do batch allocs and overfill this bin as a result, we only
* allow one batch allocation at a time for a bin. This bool tracks
* whether or not some thread is already batch allocating.
*
* Eventually, the right answer may be a smarter sharding policy for the
* bins (e.g. a mutex per bin, which would also be more scalable
* generally; the batch-allocating thread could hold it while
* batch-allocating).
* Protects the data members of the bin.
*/
bool being_batch_filled;
malloc_mutex_t mtx;
/*
* Number of bytes in this particular bin (as opposed to the
* sec_shard_t's bytes_cur. This isn't user visible or reported in
* stats; rather, it allows us to quickly determine the change in the
* centralized counter when flushing.
* Number of bytes in this particular bin.
*/
size_t bytes_cur;
edata_list_active_t freelist;
};
typedef struct sec_shard_s sec_shard_t;
struct sec_shard_s {
/*
* We don't keep per-bin mutexes, even though that would allow more
* sharding; this allows global cache-eviction, which in turn allows for
* better balancing across free lists.
*/
malloc_mutex_t mtx;
/*
* A SEC may need to be shut down (i.e. flushed of its contents and
* prevented from further caching). To avoid tricky synchronization
* issues, we just track enabled-status in each shard, guarded by a
* mutex. In practice, this is only ever checked during brief races,
* since the arena-level atomic boolean tracking HPA enabled-ness means
* that we won't go down these pathways very often after custom extent
* hooks are installed.
*/
bool enabled;
sec_bin_t *bins;
/* Number of bytes in all bins in the shard. */
size_t bytes_cur;
/* The next pszind to flush in the flush-some pathways. */
pszind_t to_flush_next;
sec_bin_stats_t stats;
};
typedef struct sec_s sec_t;
struct sec_s {
pai_t pai;
pai_t *fallback;
sec_opts_t opts;
sec_shard_t *shards;
pszind_t npsizes;
sec_opts_t opts;
sec_bin_t *bins;
pszind_t npsizes;
};
bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
const sec_opts_t *opts);
void sec_flush(tsdn_t *tsdn, sec_t *sec);
void sec_disable(tsdn_t *tsdn, sec_t *sec);
static inline bool
sec_is_used(sec_t *sec) {
return sec->opts.nshards != 0;
}
static inline bool
sec_size_supported(sec_t *sec, size_t size) {
return sec_is_used(sec) && size <= sec->opts.max_alloc;
}
/* If sec does not have extent available, it will return NULL. */
edata_t *sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size);
void sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
edata_list_active_t *result, size_t nallocs);
/*
* Upon return dalloc_list may be empty if edata is consumed by sec or non-empty
* if there are extents that need to be flushed from cache. Please note, that
* if we need to flush, extent(s) returned in the list to be deallocated
* will almost certainly not contain the one being dalloc-ed (that one will be
* considered "hot" and preserved in the cache, while "colder" ones are
* returned).
*/
void sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list);
bool sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts);
/* Fills to_flush with extents that need to be deallocated */
void sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush);
/*
* Morally, these two stats methods probably ought to be a single one (and the

43
include/jemalloc/internal/sec_opts.h
View file

@ -12,46 +12,39 @@ typedef struct sec_opts_s sec_opts_t;
struct sec_opts_s {
/*
* We don't necessarily always use all the shards; requests are
* distributed across shards [0, nshards - 1).
* distributed across shards [0, nshards - 1). Once thread picks a
* shard it will always use that one. If this value is set to 0 sec is
* not used.
*/
size_t nshards;
/*
* We'll automatically refuse to cache any objects in this sec if
* they're larger than max_alloc bytes, instead forwarding such objects
* directly to the fallback.
* they're larger than max_alloc bytes.
*/
size_t max_alloc;
/*
* Exceeding this amount of cached extents in a shard causes us to start
* flushing bins in that shard until we fall below bytes_after_flush.
* Exceeding this amount of cached extents in a bin causes us to flush
* until we are 1/4 below max_bytes.
*/
size_t max_bytes;
/*
* The number of bytes (in all bins) we flush down to when we exceed
* bytes_cur. We want this to be less than bytes_cur, because
* otherwise we could get into situations where a shard undergoing
* net-deallocation keeps bytes_cur very near to max_bytes, so that
* most deallocations get immediately forwarded to the underlying PAI
* implementation, defeating the point of the SEC.
*/
size_t bytes_after_flush;
/*
* When we can't satisfy an allocation out of the SEC because there are
* no available ones cached, we allocate multiple of that size out of
* the fallback allocator. Eventually we might want to do something
* cleverer, but for now we just grab a fixed number.
* no available ones cached, allocator will allocate a batch with extra
* batch_fill_extra extents of the same size.
*/
size_t batch_fill_extra;
};
#define SEC_OPTS_NSHARDS_DEFAULT 2
#define SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT 3
#define SEC_OPTS_MAX_ALLOC_DEFAULT ((32 * 1024) < PAGE ? PAGE : (32 * 1024))
#define SEC_OPTS_MAX_BYTES_DEFAULT \
((256 * 1024) < (4 * SEC_OPTS_MAX_ALLOC_DEFAULT) \
? (4 * SEC_OPTS_MAX_ALLOC_DEFAULT) \
: (256 * 1024))
#define SEC_OPTS_DEFAULT \
{ \
/* nshards */ \
4, /* max_alloc */ \
(32 * 1024) < PAGE ? PAGE : (32 * 1024), /* max_bytes */ \
256 * 1024, /* bytes_after_flush */ \
128 * 1024, /* batch_fill_extra */ \
0 \
}
{SEC_OPTS_NSHARDS_DEFAULT, SEC_OPTS_MAX_ALLOC_DEFAULT, \
SEC_OPTS_MAX_BYTES_DEFAULT, SEC_OPTS_BATCH_FILL_EXTRA_DEFAULT}
#endif /* JEMALLOC_INTERNAL_SEC_OPTS_H */

2
include/jemalloc/internal/witness.h
View file

@ -46,7 +46,7 @@ enum witness_rank_e {
WITNESS_RANK_DECAY = WITNESS_RANK_CORE,
WITNESS_RANK_TCACHE_QL,
WITNESS_RANK_SEC_SHARD,
WITNESS_RANK_SEC_BIN,
WITNESS_RANK_EXTENT_GROW,
WITNESS_RANK_HPA_SHARD_GROW = WITNESS_RANK_EXTENT_GROW,

1
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
View file

@ -74,7 +74,6 @@
<ClCompile Include="..\..\..\..\src\nstime.c" />
<ClCompile Include="..\..\..\..\src\pa.c" />
<ClCompile Include="..\..\..\..\src\pa_extra.c" />
<ClCompile Include="..\..\..\..\src\pai.c" />
<ClCompile Include="..\..\..\..\src\pac.c" />
<ClCompile Include="..\..\..\..\src\pages.c" />
<ClCompile Include="..\..\..\..\src\peak_event.c" />

3
msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
View file

@ -106,9 +106,6 @@
<ClCompile Include="..\..\..\..\src\pa_extra.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pai.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pac.c">
<Filter>Source Files</Filter>
</ClCompile>

1
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
View file

@ -74,7 +74,6 @@
<ClCompile Include="..\..\..\..\src\nstime.c" />
<ClCompile Include="..\..\..\..\src\pa.c" />
<ClCompile Include="..\..\..\..\src\pa_extra.c" />
<ClCompile Include="..\..\..\..\src\pai.c" />
<ClCompile Include="..\..\..\..\src\pac.c" />
<ClCompile Include="..\..\..\..\src\pages.c" />
<ClCompile Include="..\..\..\..\src\peak_event.c" />

3
msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
View file

@ -106,9 +106,6 @@
<ClCompile Include="..\..\..\..\src\pa_extra.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pai.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pac.c">
<Filter>Source Files</Filter>
</ClCompile>

1
msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
View file

@ -74,7 +74,6 @@
<ClCompile Include="..\..\..\..\src\nstime.c" />
<ClCompile Include="..\..\..\..\src\pa.c" />
<ClCompile Include="..\..\..\..\src\pa_extra.c" />
<ClCompile Include="..\..\..\..\src\pai.c" />
<ClCompile Include="..\..\..\..\src\pac.c" />
<ClCompile Include="..\..\..\..\src\pages.c" />
<ClCompile Include="..\..\..\..\src\peak_event.c" />

3
msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
View file

@ -106,9 +106,6 @@
<ClCompile Include="..\..\..\..\src\pa_extra.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pai.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pac.c">
<Filter>Source Files</Filter>
</ClCompile>

1
msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
View file

@ -74,7 +74,6 @@
<ClCompile Include="..\..\..\..\src\nstime.c" />
<ClCompile Include="..\..\..\..\src\pa.c" />
<ClCompile Include="..\..\..\..\src\pa_extra.c" />
<ClCompile Include="..\..\..\..\src\pai.c" />
<ClCompile Include="..\..\..\..\src\pac.c" />
<ClCompile Include="..\..\..\..\src\pages.c" />
<ClCompile Include="..\..\..\..\src\peak_event.c" />

3
msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
View file

@ -106,9 +106,6 @@
<ClCompile Include="..\..\..\..\src\pa_extra.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pai.c">
<Filter>Source Files</Filter>
</ClCompile>
<ClCompile Include="..\..\..\..\src\pac.c">
<Filter>Source Files</Filter>
</ClCompile>

6
src/arena.c
View file

@ -89,7 +89,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms,
size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats,
bin_stats_data_t *bstats, arena_stats_large_t *lstats, pac_estats_t *estats,
hpa_shard_stats_t *hpastats, sec_stats_t *secstats) {
hpa_shard_stats_t *hpastats) {
cassert(config_stats);
arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms,
@ -159,7 +159,7 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads,
}
pa_shard_stats_merge(tsdn, &arena->pa_shard, &astats->pa_shard_stats,
estats, hpastats, secstats, &astats->resident);
estats, hpastats, &astats->resident);
LOCKEDINT_MTX_UNLOCK(tsdn, arena->stats.mtx);
@ -529,7 +529,7 @@ arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, bool all) {
* as possible", including flushing any caches (for situations
* like thread death, or manual purge calls).
*/
sec_flush(tsdn, &arena->pa_shard.hpa_sec);
pa_shard_flush(tsdn, &arena->pa_shard);
}
if (arena_decay_dirty(tsdn, arena, is_background_thread, all)) {
return;

31
src/ctl.c
View file

@ -115,7 +115,6 @@ CTL_PROTO(opt_hpa_dirty_mult)
CTL_PROTO(opt_hpa_sec_nshards)
CTL_PROTO(opt_hpa_sec_max_alloc)
CTL_PROTO(opt_hpa_sec_max_bytes)
CTL_PROTO(opt_hpa_sec_bytes_after_flush)
CTL_PROTO(opt_hpa_sec_batch_fill_extra)
CTL_PROTO(opt_huge_arena_pac_thp)
CTL_PROTO(opt_metadata_thp)
@ -339,6 +338,11 @@ CTL_PROTO(stats_arenas_i_tcache_stashed_bytes)
CTL_PROTO(stats_arenas_i_resident)
CTL_PROTO(stats_arenas_i_abandoned_vm)
CTL_PROTO(stats_arenas_i_hpa_sec_bytes)
CTL_PROTO(stats_arenas_i_hpa_sec_hits)
CTL_PROTO(stats_arenas_i_hpa_sec_misses)
CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_flush)
CTL_PROTO(stats_arenas_i_hpa_sec_dalloc_noflush)
CTL_PROTO(stats_arenas_i_hpa_sec_overfills)
INDEX_PROTO(stats_arenas_i)
CTL_PROTO(stats_allocated)
CTL_PROTO(stats_active)
@ -486,7 +490,6 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
{NAME("hpa_sec_nshards"), CTL(opt_hpa_sec_nshards)},
{NAME("hpa_sec_max_alloc"), CTL(opt_hpa_sec_max_alloc)},
{NAME("hpa_sec_max_bytes"), CTL(opt_hpa_sec_max_bytes)},
{NAME("hpa_sec_bytes_after_flush"), CTL(opt_hpa_sec_bytes_after_flush)},
{NAME("hpa_sec_batch_fill_extra"), CTL(opt_hpa_sec_batch_fill_extra)},
{NAME("huge_arena_pac_thp"), CTL(opt_huge_arena_pac_thp)},
{NAME("metadata_thp"), CTL(opt_metadata_thp)},
@ -826,6 +829,12 @@ static const ctl_named_node_t stats_arenas_i_node[] = {
{NAME("resident"), CTL(stats_arenas_i_resident)},
{NAME("abandoned_vm"), CTL(stats_arenas_i_abandoned_vm)},
{NAME("hpa_sec_bytes"), CTL(stats_arenas_i_hpa_sec_bytes)},
{NAME("hpa_sec_hits"), CTL(stats_arenas_i_hpa_sec_hits)},
{NAME("hpa_sec_misses"), CTL(stats_arenas_i_hpa_sec_misses)},
{NAME("hpa_sec_dalloc_noflush"),
CTL(stats_arenas_i_hpa_sec_dalloc_noflush)},
{NAME("hpa_sec_dalloc_flush"), CTL(stats_arenas_i_hpa_sec_dalloc_flush)},
{NAME("hpa_sec_overfills"), CTL(stats_arenas_i_hpa_sec_overfills)},
{NAME("small"), CHILD(named, stats_arenas_i_small)},
{NAME("large"), CHILD(named, stats_arenas_i_large)},
{NAME("bins"), CHILD(indexed, stats_arenas_i_bins)},
@ -1066,7 +1075,7 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) {
&ctl_arena->pdirty, &ctl_arena->pmuzzy,
&ctl_arena->astats->astats, ctl_arena->astats->bstats,
ctl_arena->astats->lstats, ctl_arena->astats->estats,
&ctl_arena->astats->hpastats, &ctl_arena->astats->secstats);
&ctl_arena->astats->hpastats);
for (i = 0; i < SC_NBINS; i++) {
bin_stats_t *bstats =
@ -1258,7 +1267,6 @@ ctl_arena_stats_sdmerge(
/* Merge HPA stats. */
hpa_shard_stats_accum(&sdstats->hpastats, &astats->hpastats);
sec_stats_accum(&sdstats->secstats, &astats->secstats);
}
}
@ -2175,11 +2183,8 @@ CTL_RO_NL_GEN(opt_hpa_slab_max_alloc, opt_hpa_opts.slab_max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_nshards, opt_hpa_sec_opts.nshards, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_alloc, opt_hpa_sec_opts.max_alloc, size_t)
CTL_RO_NL_GEN(opt_hpa_sec_max_bytes, opt_hpa_sec_opts.max_bytes, size_t)
CTL_RO_NL_GEN(
opt_hpa_sec_bytes_after_flush, opt_hpa_sec_opts.bytes_after_flush, size_t)
CTL_RO_NL_GEN(
opt_hpa_sec_batch_fill_extra, opt_hpa_sec_opts.batch_fill_extra, size_t)
CTL_RO_NL_GEN(opt_huge_arena_pac_thp, opt_huge_arena_pac_thp, bool)
CTL_RO_NL_GEN(
opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], const char *)
@ -3869,7 +3874,17 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_abandoned_vm,
size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_bytes,
arenas_i(mib[2])->astats->secstats.bytes, size_t)
arenas_i(mib[2])->astats->hpastats.secstats.bytes, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_hits,
arenas_i(mib[2])->astats->hpastats.secstats.total.nhits, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_misses,
arenas_i(mib[2])->astats->hpastats.secstats.total.nmisses, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_flush,
arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_flush, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_dalloc_noflush,
arenas_i(mib[2])->astats->hpastats.secstats.total.ndalloc_noflush, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_hpa_sec_overfills,
arenas_i(mib[2])->astats->hpastats.secstats.total.noverfills, size_t)
CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated,
arenas_i(mib[2])->astats->allocated_small, size_t)

144
src/hpa.c
View file

@ -11,19 +11,17 @@
static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
static size_t hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size,
size_t nallocs, edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated);
static bool hpa_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool hpa_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void hpa_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated);
static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
static void hpa_dalloc_batch(tsdn_t *tsdn, pai_t *self,
edata_list_active_t *list, bool *deferred_work_generated);
const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
bool opt_experimental_hpa_start_huge_if_thp_always = true;
@ -74,9 +72,9 @@ hpa_do_consistency_checks(hpa_shard_t *shard) {
}
bool
hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts) {
hpa_shard_init(tsdn_t *tsdn, hpa_shard_t *shard, hpa_central_t *central,
emap_t *emap, base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts, const sec_opts_t *sec_opts) {
/* malloc_conf processing should have filtered out these cases. */
assert(hpa_supported());
bool err;
@ -118,13 +116,16 @@ hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
* operating on corrupted data.
*/
shard->pai.alloc = &hpa_alloc;
shard->pai.alloc_batch = &hpa_alloc_batch;
shard->pai.expand = &hpa_expand;
shard->pai.shrink = &hpa_shrink;
shard->pai.dalloc = &hpa_dalloc;
shard->pai.dalloc_batch = &hpa_dalloc_batch;
shard->pai.time_until_deferred_work = &hpa_time_until_deferred_work;
err = sec_init(tsdn, &shard->sec, base, sec_opts);
if (err) {
return true;
}
hpa_do_consistency_checks(shard);
return false;
@ -151,6 +152,7 @@ hpa_shard_stats_accum(hpa_shard_stats_t *dst, hpa_shard_stats_t *src) {
psset_stats_accum(&dst->psset_stats, &src->psset_stats);
hpa_shard_nonderived_stats_accum(
&dst->nonderived_stats, &src->nonderived_stats);
sec_stats_accum(&dst->secstats, &src->secstats);
}
void
@ -164,6 +166,8 @@ hpa_shard_stats_merge(
hpa_shard_nonderived_stats_accum(&dst->nonderived_stats, &shard->stats);
malloc_mutex_unlock(tsdn, &shard->mtx);
malloc_mutex_unlock(tsdn, &shard->grow_mtx);
sec_stats_merge(tsdn, &shard->sec, &dst->secstats);
}
static bool
@ -825,37 +829,9 @@ hpa_from_pai(pai_t *self) {
return (hpa_shard_t *)self;
}
static size_t
hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated) {
assert(nallocs > 0);
assert((size & PAGE_MASK) == 0);
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
hpa_shard_t *shard = hpa_from_pai(self);
/*
* frequent_use here indicates this request comes from the arena bins,
* in which case it will be split into slabs, and therefore there is no
* intrinsic slack in the allocation (the entire range of allocated size
* will be accessed).
*
* In this case bypass the slab_max_alloc limit (if still within the
* huge page size). These requests do not concern internal
* fragmentation with huge pages (again, the full size will be used).
*/
if (!(frequent_reuse && size <= HUGEPAGE)
&& (size > shard->opts.slab_max_alloc)) {
return 0;
}
size_t nsuccess = hpa_alloc_batch_psset(
tsdn, shard, size, nallocs, results, deferred_work_generated);
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
static void
hpa_assert_results(
tsdn_t *tsdn, hpa_shard_t *shard, edata_list_active_t *results) {
/*
* Guard the sanity checks with config_debug because the loop cannot be
* proven non-circular by the compiler, even if everything within the
@ -876,7 +852,6 @@ hpa_alloc_batch(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
assert(edata_base_get(edata) != NULL);
}
}
return nsuccess;
}
static edata_t *
@ -891,16 +866,52 @@ hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
if (alignment > PAGE || zero) {
return NULL;
}
hpa_shard_t *shard = hpa_from_pai(self);
/*
* An alloc with alignment == PAGE and zero == false is equivalent to a
* batch alloc of 1. Just do that, so we can share code.
* frequent_use here indicates this request comes from the arena bins,
* in which case it will be split into slabs, and therefore there is no
* intrinsic slack in the allocation (the entire range of allocated size
* will be accessed).
*
* In this case bypass the slab_max_alloc limit (if still within the
* huge page size). These requests do not concern internal
* fragmentation with huge pages (again, the full size will be used).
*/
if (!(frequent_reuse && size <= HUGEPAGE)
&& (size > shard->opts.slab_max_alloc)) {
return NULL;
}
edata_t *edata = sec_alloc(tsdn, &shard->sec, size);
if (edata != NULL) {
return edata;
}
size_t nallocs = sec_size_supported(&shard->sec, size)
? shard->sec.opts.batch_fill_extra + 1
: 1;
edata_list_active_t results;
edata_list_active_init(&results);
size_t nallocs = hpa_alloc_batch(tsdn, self, size, /* nallocs */ 1,
&results, frequent_reuse, deferred_work_generated);
assert(nallocs == 0 || nallocs == 1);
edata_t *edata = edata_list_active_first(&results);
size_t nsuccess = hpa_alloc_batch_psset(
tsdn, shard, size, nallocs, &results, deferred_work_generated);
hpa_assert_results(tsdn, shard, &results);
edata = edata_list_active_first(&results);
if (edata != NULL) {
edata_list_active_remove(&results, edata);
assert(nsuccess > 0);
nsuccess--;
}
if (nsuccess > 0) {
assert(sec_size_supported(&shard->sec, size));
sec_fill(tsdn, &shard->sec, size, &results, nsuccess);
/* Unlikely rollback in case of overfill */
if (!edata_list_active_empty(&results)) {
hpa_dalloc_batch(
tsdn, self, &results, deferred_work_generated);
}
}
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
return edata;
}
@ -996,10 +1007,19 @@ static void
hpa_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
assert(!edata_guarded_get(edata));
/* Just a dalloc_batch of size 1; this lets us share logic. */
edata_list_active_t dalloc_list;
edata_list_active_init(&dalloc_list);
edata_list_active_append(&dalloc_list, edata);
hpa_shard_t *shard = hpa_from_pai(self);
sec_dalloc(tsdn, &shard->sec, &dalloc_list);
if (edata_list_active_empty(&dalloc_list)) {
/* sec consumed the pointer */
*deferred_work_generated = false;
return;
}
/* We may have more than one pointer to flush now */
hpa_dalloc_batch(tsdn, self, &dalloc_list, deferred_work_generated);
}
@ -1063,15 +1083,32 @@ hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self) {
return time_ns;
}
static void
hpa_sec_flush_impl(tsdn_t *tsdn, hpa_shard_t *shard) {
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
sec_flush(tsdn, &shard->sec, &to_flush);
bool deferred_work_generated;
hpa_dalloc_batch(
tsdn, (pai_t *)shard, &to_flush, &deferred_work_generated);
}
void
hpa_shard_disable(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
hpa_sec_flush_impl(tsdn, shard);
malloc_mutex_lock(tsdn, &shard->mtx);
edata_cache_fast_disable(tsdn, &shard->ecf);
malloc_mutex_unlock(tsdn, &shard->mtx);
}
void
hpa_shard_flush(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_sec_flush_impl(tsdn, shard);
}
static void
hpa_shard_assert_stats_empty(psset_bin_stats_t *bin_stats) {
assert(bin_stats->npageslabs == 0);
@ -1093,6 +1130,7 @@ hpa_assert_empty(tsdn_t *tsdn, hpa_shard_t *shard, psset_t *psset) {
void
hpa_shard_destroy(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
hpa_shard_flush(tsdn, shard);
/*
* By the time we're here, the arena code should have dalloc'd all the
* active extents, which means we should have eventually evicted
@ -1137,6 +1175,12 @@ hpa_shard_do_deferred_work(tsdn_t *tsdn, hpa_shard_t *shard) {
malloc_mutex_unlock(tsdn, &shard->mtx);
}
void
hpa_shard_prefork2(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
sec_prefork2(tsdn, &shard->sec);
}
void
hpa_shard_prefork3(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
@ -1155,6 +1199,7 @@ void
hpa_shard_postfork_parent(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
sec_postfork_parent(tsdn, &shard->sec);
malloc_mutex_postfork_parent(tsdn, &shard->grow_mtx);
malloc_mutex_postfork_parent(tsdn, &shard->mtx);
}
@ -1163,6 +1208,7 @@ void
hpa_shard_postfork_child(tsdn_t *tsdn, hpa_shard_t *shard) {
hpa_do_consistency_checks(shard);
sec_postfork_child(tsdn, &shard->sec);
malloc_mutex_postfork_child(tsdn, &shard->grow_mtx);
malloc_mutex_postfork_child(tsdn, &shard->mtx);
}

17
src/jemalloc.c
View file

@ -1013,6 +1013,15 @@ malloc_conf_error(
/* However, tolerate experimental features. */
return;
}
const char *deprecated[] = {"hpa_sec_bytes_after_flush"};
const size_t deprecated_cnt = (sizeof(deprecated)
/ sizeof(deprecated[0]));
for (size_t i = 0; i < deprecated_cnt; ++i) {
if (strncmp(k, deprecated[i], strlen(deprecated[i])) == 0) {
/* Tolerate deprecated features. */
return;
}
}
had_conf_error = true;
}
@ -1685,7 +1694,6 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
}
CONF_CONTINUE;
}
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.nshards,
"hpa_sec_nshards", 0, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
@ -1694,13 +1702,10 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
USIZE_GROW_SLOW_THRESHOLD, CONF_CHECK_MIN,
CONF_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.max_bytes,
"hpa_sec_max_bytes", PAGE, 0, CONF_CHECK_MIN,
CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.bytes_after_flush,
"hpa_sec_bytes_after_flush", PAGE, 0,
"hpa_sec_max_bytes", SEC_OPTS_MAX_BYTES_DEFAULT, 0,
CONF_CHECK_MIN, CONF_DONT_CHECK_MAX, true);
CONF_HANDLE_SIZE_T(opt_hpa_sec_opts.batch_fill_extra,
"hpa_sec_batch_fill_extra", 0, HUGEPAGE_PAGES,
"hpa_sec_batch_fill_extra", 1, HUGEPAGE_PAGES,
CONF_CHECK_MIN, CONF_CHECK_MAX, true);
if (CONF_MATCH("slab_sizes")) {

22
src/pa.c
View file

@ -67,12 +67,9 @@ pa_shard_init(tsdn_t *tsdn, pa_shard_t *shard, pa_central_t *central,
bool
pa_shard_enable_hpa(tsdn_t *tsdn, pa_shard_t *shard,
const hpa_shard_opts_t *hpa_opts, const sec_opts_t *hpa_sec_opts) {
if (hpa_shard_init(&shard->hpa_shard, &shard->central->hpa, shard->emap,
shard->base, &shard->edata_cache, shard->ind, hpa_opts)) {
return true;
}
if (sec_init(tsdn, &shard->hpa_sec, shard->base, &shard->hpa_shard.pai,
hpa_sec_opts)) {
if (hpa_shard_init(tsdn, &shard->hpa_shard, &shard->central->hpa,
shard->emap, shard->base, &shard->edata_cache, shard->ind,
hpa_opts, hpa_sec_opts)) {
return true;
}
shard->ever_used_hpa = true;
@ -85,7 +82,6 @@ void
pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
atomic_store_b(&shard->use_hpa, false, ATOMIC_RELAXED);
if (shard->ever_used_hpa) {
sec_disable(tsdn, &shard->hpa_sec);
hpa_shard_disable(tsdn, &shard->hpa_shard);
}
}
@ -93,8 +89,13 @@ pa_shard_disable_hpa(tsdn_t *tsdn, pa_shard_t *shard) {
void
pa_shard_reset(tsdn_t *tsdn, pa_shard_t *shard) {
atomic_store_zu(&shard->nactive, 0, ATOMIC_RELAXED);
pa_shard_flush(tsdn, shard);
}
void
pa_shard_flush(tsdn_t *tsdn, pa_shard_t *shard) {
if (shard->ever_used_hpa) {
sec_flush(tsdn, &shard->hpa_sec);
hpa_shard_flush(tsdn, &shard->hpa_shard);
}
}
@ -107,7 +108,6 @@ void
pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
pac_destroy(tsdn, &shard->pac);
if (shard->ever_used_hpa) {
sec_flush(tsdn, &shard->hpa_sec);
hpa_shard_destroy(tsdn, &shard->hpa_shard);
}
}
@ -115,7 +115,7 @@ pa_shard_destroy(tsdn_t *tsdn, pa_shard_t *shard) {
static pai_t *
pa_get_pai(pa_shard_t *shard, edata_t *edata) {
return (edata_pai_get(edata) == EXTENT_PAI_PAC ? &shard->pac.pai
: &shard->hpa_sec.pai);
: &shard->hpa_shard.pai);
}
edata_t *
@ -128,7 +128,7 @@ pa_alloc(tsdn_t *tsdn, pa_shard_t *shard, size_t size, size_t alignment,
edata_t *edata = NULL;
if (!guarded && pa_shard_uses_hpa(shard)) {
edata = pai_alloc(tsdn, &shard->hpa_sec.pai, size, alignment,
edata = pai_alloc(tsdn, &shard->hpa_shard.pai, size, alignment,
zero, /* guarded */ false, slab, deferred_work_generated);
}
/*

10
src/pa_extra.c
View file

@ -17,7 +17,7 @@ pa_shard_prefork0(tsdn_t *tsdn, pa_shard_t *shard) {
void
pa_shard_prefork2(tsdn_t *tsdn, pa_shard_t *shard) {
if (shard->ever_used_hpa) {
sec_prefork2(tsdn, &shard->hpa_sec);
hpa_shard_prefork2(tsdn, &shard->hpa_shard);
}
}
@ -54,7 +54,6 @@ pa_shard_postfork_parent(tsdn_t *tsdn, pa_shard_t *shard) {
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_parent(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
sec_postfork_parent(tsdn, &shard->hpa_sec);
hpa_shard_postfork_parent(tsdn, &shard->hpa_shard);
}
}
@ -69,7 +68,6 @@ pa_shard_postfork_child(tsdn_t *tsdn, pa_shard_t *shard) {
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_dirty.mtx);
malloc_mutex_postfork_child(tsdn, &shard->pac.decay_muzzy.mtx);
if (shard->ever_used_hpa) {
sec_postfork_child(tsdn, &shard->hpa_sec);
hpa_shard_postfork_child(tsdn, &shard->hpa_shard);
}
}
@ -104,8 +102,7 @@ pa_shard_basic_stats_merge(
void
pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_stats_t *pa_shard_stats_out, pac_estats_t *estats_out,
hpa_shard_stats_t *hpa_stats_out, sec_stats_t *sec_stats_out,
size_t *resident) {
hpa_shard_stats_t *hpa_stats_out, size_t *resident) {
cassert(config_stats);
pa_shard_stats_out->pac_stats.retained +=
@ -170,7 +167,6 @@ pa_shard_stats_merge(tsdn_t *tsdn, pa_shard_t *shard,
if (shard->ever_used_hpa) {
hpa_shard_stats_merge(tsdn, &shard->hpa_shard, hpa_stats_out);
sec_stats_merge(tsdn, &shard->hpa_sec, sec_stats_out);
}
}
@ -204,7 +200,7 @@ pa_shard_mtx_stats_read(tsdn_t *tsdn, pa_shard_t *shard,
pa_shard_mtx_stats_read_single(tsdn, mutex_prof_data,
&shard->hpa_shard.grow_mtx,
arena_prof_mutex_hpa_shard_grow);
sec_mutex_stats_read(tsdn, &shard->hpa_sec,
sec_mutex_stats_read(tsdn, &shard->hpa_shard.sec,
&mutex_prof_data[arena_prof_mutex_hpa_sec]);
}
}

8
src/pac.c
View file

@ -97,11 +97,9 @@ pac_init(tsdn_t *tsdn, pac_t *pac, base_t *base, emap_t *emap,
atomic_store_zu(&pac->extent_sn_next, 0, ATOMIC_RELAXED);
pac->pai.alloc = &pac_alloc_impl;
pac->pai.alloc_batch = &pai_alloc_batch_default;
pac->pai.expand = &pac_expand_impl;
pac->pai.shrink = &pac_shrink_impl;
pac->pai.dalloc = &pac_dalloc_impl;
pac->pai.dalloc_batch = &pai_dalloc_batch_default;
pac->pai.time_until_deferred_work = &pac_time_until_deferred_work;
return false;
@ -449,8 +447,8 @@ decay_with_process_madvise(edata_list_inactive_t *decay_extents) {
size_t cur = 0, total_bytes = 0;
for (edata_t *edata = edata_list_inactive_first(decay_extents);
edata != NULL;
edata = edata_list_inactive_next(decay_extents, edata)) {
edata != NULL;
edata = edata_list_inactive_next(decay_extents, edata)) {
size_t pages_bytes = edata_size_get(edata);
vec[cur].iov_base = edata_base_get(edata);
vec[cur].iov_len = pages_bytes;
@ -511,7 +509,7 @@ pac_decay_stashed(tsdn_t *tsdn, pac_t *pac, decay_t *decay,
}
for (edata_t *edata = edata_list_inactive_first(decay_extents);
edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
edata != NULL; edata = edata_list_inactive_first(decay_extents)) {
edata_list_inactive_remove(decay_extents, edata);
size_t size = edata_size_get(edata);

32
src/pai.c
View file

@ -1,32 +0,0 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/jemalloc_internal_includes.h"
size_t
pai_alloc_batch_default(tsdn_t *tsdn, pai_t *self, size_t size, size_t nallocs,
edata_list_active_t *results, bool frequent_reuse,
bool *deferred_work_generated) {
for (size_t i = 0; i < nallocs; i++) {
bool deferred_by_alloc = false;
edata_t *edata = pai_alloc(tsdn, self, size, PAGE,
/* zero */ false, /* guarded */ false, frequent_reuse,
&deferred_by_alloc);
*deferred_work_generated |= deferred_by_alloc;
if (edata == NULL) {
return i;
}
edata_list_active_append(results, edata);
}
return nallocs;
}
void
pai_dalloc_batch_default(tsdn_t *tsdn, pai_t *self, edata_list_active_t *list,
bool *deferred_work_generated) {
edata_t *edata;
while ((edata = edata_list_active_first(list)) != NULL) {
bool deferred_by_dalloc = false;
edata_list_active_remove(list, edata);
pai_dalloc(tsdn, self, edata, &deferred_by_dalloc);
*deferred_work_generated |= deferred_by_dalloc;
}
}

564
src/sec.c
View file

@ -4,95 +4,56 @@
#include "jemalloc/internal/sec.h"
#include "jemalloc/internal/jemalloc_probe.h"
static edata_t *sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
static bool sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool zero, bool *deferred_work_generated);
static bool sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata,
size_t old_size, size_t new_size, bool *deferred_work_generated);
static void sec_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated);
static void
static bool
sec_bin_init(sec_bin_t *bin) {
bin->being_batch_filled = false;
bin->bytes_cur = 0;
sec_bin_stats_init(&bin->stats);
edata_list_active_init(&bin->freelist);
bool err = malloc_mutex_init(&bin->mtx, "sec_bin", WITNESS_RANK_SEC_BIN,
malloc_mutex_rank_exclusive);
if (err) {
return true;
}
return false;
}
bool
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, pai_t *fallback,
const sec_opts_t *opts) {
sec_init(tsdn_t *tsdn, sec_t *sec, base_t *base, const sec_opts_t *opts) {
sec->opts = *opts;
if (opts->nshards == 0) {
return false;
}
assert(opts->max_alloc >= PAGE);
/*
* Same as tcache, sec do not cache allocs/dallocs larger than
* USIZE_GROW_SLOW_THRESHOLD because the usize above this increases
* by PAGE and the number of usizes is too large.
*/
assert(!sz_large_size_classes_disabled()
|| opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
assert(opts->max_alloc <= USIZE_GROW_SLOW_THRESHOLD);
size_t max_alloc = PAGE_FLOOR(opts->max_alloc);
pszind_t npsizes = sz_psz2ind(max_alloc) + 1;
size_t sz_shards = opts->nshards * sizeof(sec_shard_t);
size_t sz_bins = opts->nshards * (size_t)npsizes * sizeof(sec_bin_t);
size_t sz_alloc = sz_shards + sz_bins;
void *dynalloc = base_alloc(tsdn, base, sz_alloc, CACHELINE);
size_t ntotal_bins = opts->nshards * (size_t)npsizes;
size_t sz_bins = sizeof(sec_bin_t) * ntotal_bins;
void *dynalloc = base_alloc(tsdn, base, sz_bins, CACHELINE);
if (dynalloc == NULL) {
return true;
}
sec_shard_t *shard_cur = (sec_shard_t *)dynalloc;
sec->shards = shard_cur;
sec_bin_t *bin_cur = (sec_bin_t *)&shard_cur[opts->nshards];
/* Just for asserts, below. */
sec_bin_t *bin_start = bin_cur;
for (size_t i = 0; i < opts->nshards; i++) {
sec_shard_t *shard = shard_cur;
shard_cur++;
bool err = malloc_mutex_init(&shard->mtx, "sec_shard",
WITNESS_RANK_SEC_SHARD, malloc_mutex_rank_exclusive);
if (err) {
sec->bins = (sec_bin_t *)dynalloc;
for (pszind_t j = 0; j < ntotal_bins; j++) {
if (sec_bin_init(&sec->bins[j])) {
return true;
}
shard->enabled = true;
shard->bins = bin_cur;
for (pszind_t j = 0; j < npsizes; j++) {
sec_bin_init(&shard->bins[j]);
bin_cur++;
}
shard->bytes_cur = 0;
shard->to_flush_next = 0;
}
/*
* Should have exactly matched the bin_start to the first unused byte
* after the shards.
*/
assert((void *)shard_cur == (void *)bin_start);
/* And the last bin to use up the last bytes of the allocation. */
assert((char *)bin_cur == ((char *)dynalloc + sz_alloc));
sec->fallback = fallback;
sec->opts = *opts;
sec->npsizes = npsizes;
/*
* Initialize these last so that an improper use of an SEC whose
* initialization failed will segfault in an easy-to-spot way.
*/
sec->pai.alloc = &sec_alloc;
sec->pai.alloc_batch = &pai_alloc_batch_default;
sec->pai.expand = &sec_expand;
sec->pai.shrink = &sec_shrink;
sec->pai.dalloc = &sec_dalloc;
sec->pai.dalloc_batch = &pai_dalloc_batch_default;
return false;
}
static sec_shard_t *
static uint8_t
sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
/*
* Eventually, we should implement affinity, tracking source shard using
@ -100,7 +61,7 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
* distribute across all shards.
*/
if (tsdn_null(tsdn)) {
return &sec->shards[0];
return 0;
}
tsd_t *tsd = tsdn_tsd(tsdn);
uint8_t *idxp = tsd_sec_shardp_get(tsd);
@ -118,284 +79,252 @@ sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
assert(idx < (uint32_t)sec->opts.nshards);
*idxp = (uint8_t)idx;
}
return &sec->shards[*idxp];
return *idxp;
}
/*
* Perhaps surprisingly, this can be called on the alloc pathways; if we hit an
* empty cache, we'll try to fill it, which can push the shard over it's limit.
*/
static void
sec_flush_some_and_unlock(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
while (shard->bytes_cur > sec->opts.bytes_after_flush) {
/* Pick a victim. */
sec_bin_t *bin = &shard->bins[shard->to_flush_next];
/* Update our victim-picking state. */
shard->to_flush_next++;
if (shard->to_flush_next == sec->npsizes) {
shard->to_flush_next = 0;
}
assert(shard->bytes_cur >= bin->bytes_cur);
if (bin->bytes_cur != 0) {
shard->bytes_cur -= bin->bytes_cur;
bin->bytes_cur = 0;
edata_list_active_concat(&to_flush, &bin->freelist);
}
/*
* Either bin->bytes_cur was 0, in which case we didn't touch
* the bin list but it should be empty anyways (or else we
* missed a bytes_cur update on a list modification), or it
* *was* 0 and we emptied it ourselves. Either way, it should
* be empty now.
*/
assert(edata_list_active_empty(&bin->freelist));
}
malloc_mutex_unlock(tsdn, &shard->mtx);
bool deferred_work_generated = false;
pai_dalloc_batch(
tsdn, sec->fallback, &to_flush, &deferred_work_generated);
static sec_bin_t *
sec_bin_pick(sec_t *sec, uint8_t shard, pszind_t pszind) {
assert(shard < sec->opts.nshards);
size_t ind = (size_t)shard * sec->npsizes + pszind;
assert(ind < sec->npsizes * sec->opts.nshards);
return &sec->bins[ind];
}
static edata_t *
sec_shard_alloc_locked(
tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, sec_bin_t *bin) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
if (!shard->enabled) {
return NULL;
}
sec_bin_alloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size) {
malloc_mutex_assert_owner(tsdn, &bin->mtx);
edata_t *edata = edata_list_active_first(&bin->freelist);
if (edata != NULL) {
assert(!edata_list_active_empty(&bin->freelist));
edata_list_active_remove(&bin->freelist, edata);
assert(edata_size_get(edata) <= bin->bytes_cur);
bin->bytes_cur -= edata_size_get(edata);
assert(edata_size_get(edata) <= shard->bytes_cur);
shard->bytes_cur -= edata_size_get(edata);
size_t sz = edata_size_get(edata);
assert(sz <= bin->bytes_cur && sz > 0);
bin->bytes_cur -= sz;
bin->stats.nhits++;
}
return edata;
}
static edata_t *
sec_batch_fill_and_alloc(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard,
sec_bin_t *bin, size_t size, bool frequent_reuse) {
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
sec_multishard_trylock_alloc(
tsdn_t *tsdn, sec_t *sec, size_t size, pszind_t pszind) {
assert(sec->opts.nshards > 0);
edata_list_active_t result;
edata_list_active_init(&result);
bool deferred_work_generated = false;
size_t nalloc = pai_alloc_batch(tsdn, sec->fallback, size,
1 + sec->opts.batch_fill_extra, &result, frequent_reuse,
&deferred_work_generated);
edata_t *ret = edata_list_active_first(&result);
if (ret != NULL) {
edata_list_active_remove(&result, ret);
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
sec_bin_t *bin;
for (size_t i = 0; i < sec->opts.nshards; ++i) {
bin = sec_bin_pick(sec, cur_shard, pszind);
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
edata_t *edata = sec_bin_alloc_locked(
tsdn, sec, bin, size);
malloc_mutex_unlock(tsdn, &bin->mtx);
if (edata != NULL) {
JE_USDT(sec_alloc, 5, sec, bin, edata, size,
/* frequent_reuse */ 1);
return edata;
}
}
cur_shard++;
if (cur_shard == sec->opts.nshards) {
cur_shard = 0;
}
}
malloc_mutex_lock(tsdn, &shard->mtx);
bin->being_batch_filled = false;
/*
* Handle the easy case first: nothing to cache. Note that this can
* only happen in case of OOM, since sec_alloc checks the expected
* number of allocs, and doesn't bother going down the batch_fill
* pathway if there won't be anything left to cache. So to be in this
* code path, we must have asked for > 1 alloc, but only gotten 1 back.
*/
if (nalloc <= 1) {
malloc_mutex_unlock(tsdn, &shard->mtx);
return ret;
/* No bin had alloc or had the extent */
assert(cur_shard == sec_shard_pick(tsdn, sec));
bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
if (edata == NULL) {
/* Only now we know it is a miss */
bin->stats.nmisses++;
}
size_t new_cached_bytes = (nalloc - 1) * size;
edata_list_active_concat(&bin->freelist, &result);
bin->bytes_cur += new_cached_bytes;
shard->bytes_cur += new_cached_bytes;
if (shard->bytes_cur > sec->opts.max_bytes) {
sec_flush_some_and_unlock(tsdn, sec, shard);
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
}
return ret;
malloc_mutex_unlock(tsdn, &bin->mtx);
JE_USDT(sec_alloc, 5, sec, bin, edata, size, /* frequent_reuse */ 1);
return edata;
}
static edata_t *
sec_alloc(tsdn_t *tsdn, pai_t *self, size_t size, size_t alignment, bool zero,
bool guarded, bool frequent_reuse, bool *deferred_work_generated) {
edata_t *
sec_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
if (!sec_size_supported(sec, size)) {
return NULL;
}
assert((size & PAGE_MASK) == 0);
assert(!guarded);
sec_t *sec = (sec_t *)self;
if (zero || alignment > PAGE || sec->opts.nshards == 0
|| size > sec->opts.max_alloc) {
return pai_alloc(tsdn, sec->fallback, size, alignment, zero,
/* guarded */ false, frequent_reuse,
deferred_work_generated);
}
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
sec_bin_t *bin = &shard->bins[pszind];
bool do_batch_fill = false;
malloc_mutex_lock(tsdn, &shard->mtx);
edata_t *edata = sec_shard_alloc_locked(tsdn, sec, shard, bin);
if (edata == NULL) {
if (!bin->being_batch_filled
&& sec->opts.batch_fill_extra > 0) {
bin->being_batch_filled = true;
do_batch_fill = true;
/*
* If there's only one shard, skip the trylock optimization and
* go straight to the blocking lock.
*/
if (sec->opts.nshards == 1) {
sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
edata_t *edata = sec_bin_alloc_locked(tsdn, sec, bin, size);
if (edata == NULL) {
bin->stats.nmisses++;
}
malloc_mutex_unlock(tsdn, &bin->mtx);
JE_USDT(sec_alloc, 5, sec, bin, edata, size,
/* frequent_reuse */ 1);
return edata;
}
malloc_mutex_unlock(tsdn, &shard->mtx);
if (edata == NULL) {
if (do_batch_fill) {
edata = sec_batch_fill_and_alloc(
tsdn, sec, shard, bin, size, frequent_reuse);
} else {
edata = pai_alloc(tsdn, sec->fallback, size, alignment,
zero, /* guarded */ false, frequent_reuse,
deferred_work_generated);
}
}
JE_USDT(sec_alloc, 5, sec, shard, edata, size, frequent_reuse);
return edata;
}
static bool
sec_expand(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool zero, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
JE_USDT(sec_expand, 4, sec, edata, old_size, new_size);
return pai_expand(tsdn, sec->fallback, edata, old_size, new_size, zero,
deferred_work_generated);
}
static bool
sec_shrink(tsdn_t *tsdn, pai_t *self, edata_t *edata, size_t old_size,
size_t new_size, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
JE_USDT(sec_shrink, 4, sec, edata, old_size, new_size);
return pai_shrink(tsdn, sec->fallback, edata, old_size, new_size,
deferred_work_generated);
return sec_multishard_trylock_alloc(tsdn, sec, size, pszind);
}
static void
sec_flush_all_locked(tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
shard->bytes_cur = 0;
edata_list_active_t to_flush;
edata_list_active_init(&to_flush);
for (pszind_t i = 0; i < sec->npsizes; i++) {
sec_bin_t *bin = &shard->bins[i];
bin->bytes_cur = 0;
edata_list_active_concat(&to_flush, &bin->freelist);
}
sec_bin_dalloc_locked(tsdn_t *tsdn, sec_t *sec, sec_bin_t *bin, size_t size,
edata_list_active_t *dalloc_list) {
malloc_mutex_assert_owner(tsdn, &bin->mtx);
/*
* Ordinarily we would try to avoid doing the batch deallocation while
* holding the shard mutex, but the flush_all pathways only happen when
* we're disabling the HPA or resetting the arena, both of which are
* rare pathways.
*/
bool deferred_work_generated = false;
pai_dalloc_batch(
tsdn, sec->fallback, &to_flush, &deferred_work_generated);
}
static void
sec_shard_dalloc_and_unlock(
tsdn_t *tsdn, sec_t *sec, sec_shard_t *shard, edata_t *edata) {
malloc_mutex_assert_owner(tsdn, &shard->mtx);
assert(shard->bytes_cur <= sec->opts.max_bytes);
size_t size = edata_size_get(edata);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
/*
* Prepending here results in LIFO allocation per bin, which seems
* reasonable.
*/
sec_bin_t *bin = &shard->bins[pszind];
edata_list_active_prepend(&bin->freelist, edata);
bin->bytes_cur += size;
shard->bytes_cur += size;
if (shard->bytes_cur > sec->opts.max_bytes) {
/*
* We've exceeded the shard limit. We make two nods in the
* direction of fragmentation avoidance: we flush everything in
* the shard, rather than one particular bin, and we hold the
* lock while flushing (in case one of the extents we flush is
* highly preferred from a fragmentation-avoidance perspective
* in the backing allocator). This has the extra advantage of
* not requiring advanced cache balancing strategies.
*/
sec_flush_some_and_unlock(tsdn, sec, shard);
malloc_mutex_assert_not_owner(tsdn, &shard->mtx);
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
}
}
edata_t *edata = edata_list_active_first(dalloc_list);
assert(edata != NULL);
edata_list_active_remove(dalloc_list, edata);
JE_USDT(sec_dalloc, 3, sec, bin, edata);
edata_list_active_prepend(&bin->freelist, edata);
/* Single extent can be returned to SEC */
assert(edata_list_active_empty(dalloc_list));
static void
sec_dalloc(
tsdn_t *tsdn, pai_t *self, edata_t *edata, bool *deferred_work_generated) {
sec_t *sec = (sec_t *)self;
if (sec->opts.nshards == 0
|| edata_size_get(edata) > sec->opts.max_alloc) {
pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
if (bin->bytes_cur <= sec->opts.max_bytes) {
bin->stats.ndalloc_noflush++;
return;
}
sec_shard_t *shard = sec_shard_pick(tsdn, sec);
JE_USDT(sec_dalloc, 3, sec, shard, edata);
malloc_mutex_lock(tsdn, &shard->mtx);
if (shard->enabled) {
sec_shard_dalloc_and_unlock(tsdn, sec, shard, edata);
bin->stats.ndalloc_flush++;
/* we want to flush 1/4 of max_bytes */
size_t bytes_target = sec->opts.max_bytes - (sec->opts.max_bytes >> 2);
while (bin->bytes_cur > bytes_target
&& !edata_list_active_empty(&bin->freelist)) {
edata_t *cur = edata_list_active_last(&bin->freelist);
size_t sz = edata_size_get(cur);
assert(sz <= bin->bytes_cur && sz > 0);
bin->bytes_cur -= sz;
edata_list_active_remove(&bin->freelist, cur);
edata_list_active_append(dalloc_list, cur);
}
}
static void
sec_multishard_trylock_dalloc(tsdn_t *tsdn, sec_t *sec, size_t size,
pszind_t pszind, edata_list_active_t *dalloc_list) {
assert(sec->opts.nshards > 0);
/* Try to dalloc in this threads bin first */
uint8_t cur_shard = sec_shard_pick(tsdn, sec);
for (size_t i = 0; i < sec->opts.nshards; ++i) {
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
if (!malloc_mutex_trylock(tsdn, &bin->mtx)) {
sec_bin_dalloc_locked(
tsdn, sec, bin, size, dalloc_list);
malloc_mutex_unlock(tsdn, &bin->mtx);
return;
}
cur_shard++;
if (cur_shard == sec->opts.nshards) {
cur_shard = 0;
}
}
/* No bin had alloc or had the extent */
assert(cur_shard == sec_shard_pick(tsdn, sec));
sec_bin_t *bin = sec_bin_pick(sec, cur_shard, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
void
sec_dalloc(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *dalloc_list) {
if (!sec_is_used(sec)) {
return;
}
edata_t *edata = edata_list_active_first(dalloc_list);
size_t size = edata_size_get(edata);
if (size > sec->opts.max_alloc) {
return;
}
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
/*
* If there's only one shard, skip the trylock optimization and
* go straight to the blocking lock.
*/
if (sec->opts.nshards == 1) {
sec_bin_t *bin = sec_bin_pick(sec, /* shard */ 0, pszind);
malloc_mutex_lock(tsdn, &bin->mtx);
sec_bin_dalloc_locked(tsdn, sec, bin, size, dalloc_list);
malloc_mutex_unlock(tsdn, &bin->mtx);
return;
}
sec_multishard_trylock_dalloc(tsdn, sec, size, pszind, dalloc_list);
}
void
sec_fill(tsdn_t *tsdn, sec_t *sec, size_t size, edata_list_active_t *result,
size_t nallocs) {
assert((size & PAGE_MASK) == 0);
assert(sec->opts.nshards != 0 && size <= sec->opts.max_alloc);
assert(nallocs > 0);
pszind_t pszind = sz_psz2ind(size);
assert(pszind < sec->npsizes);
sec_bin_t *bin = sec_bin_pick(sec, sec_shard_pick(tsdn, sec), pszind);
malloc_mutex_assert_not_owner(tsdn, &bin->mtx);
malloc_mutex_lock(tsdn, &bin->mtx);
size_t new_cached_bytes = nallocs * size;
if (bin->bytes_cur + new_cached_bytes <= sec->opts.max_bytes) {
assert(!edata_list_active_empty(result));
edata_list_active_concat(&bin->freelist, result);
bin->bytes_cur += new_cached_bytes;
} else {
malloc_mutex_unlock(tsdn, &shard->mtx);
pai_dalloc(tsdn, sec->fallback, edata, deferred_work_generated);
/*
* Unlikely case of many threads filling at the same time and
* going above max.
*/
bin->stats.noverfills++;
while (bin->bytes_cur + size <= sec->opts.max_bytes) {
edata_t *edata = edata_list_active_first(result);
if (edata == NULL) {
break;
}
edata_list_active_remove(result, edata);
assert(size == edata_size_get(edata));
edata_list_active_append(&bin->freelist, edata);
bin->bytes_cur += size;
}
}
malloc_mutex_unlock(tsdn, &bin->mtx);
}
void
sec_flush(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
sec_flush(tsdn_t *tsdn, sec_t *sec, edata_list_active_t *to_flush) {
if (!sec_is_used(sec)) {
return;
}
}
void
sec_disable(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sec->shards[i].enabled = false;
sec_flush_all_locked(tsdn, sec, &sec->shards[i]);
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
bin->bytes_cur = 0;
edata_list_active_concat(to_flush, &bin->freelist);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
}
void
sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
if (!sec_is_used(sec)) {
return;
}
size_t sum = 0;
for (size_t i = 0; i < sec->opts.nshards; i++) {
/*
* We could save these lock acquisitions by making bytes_cur
* atomic, but stats collection is rare anyways and we expect
* the number and type of stats to get more interesting.
*/
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
sum += sec->shards[i].bytes_cur;
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
sum += bin->bytes_cur;
sec_bin_stats_accum(&stats->total, &bin->stats);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
stats->bytes += sum;
}
@ -403,31 +332,50 @@ sec_stats_merge(tsdn_t *tsdn, sec_t *sec, sec_stats_t *stats) {
void
sec_mutex_stats_read(
tsdn_t *tsdn, sec_t *sec, mutex_prof_data_t *mutex_prof_data) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_lock(tsdn, &sec->shards[i].mtx);
malloc_mutex_prof_accum(
tsdn, mutex_prof_data, &sec->shards[i].mtx);
malloc_mutex_unlock(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_lock(tsdn, &bin->mtx);
malloc_mutex_prof_accum(tsdn, mutex_prof_data, &bin->mtx);
malloc_mutex_unlock(tsdn, &bin->mtx);
}
}
void
sec_prefork2(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_prefork(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_prefork(tsdn, &bin->mtx);
}
}
void
sec_postfork_parent(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_postfork_parent(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_postfork_parent(tsdn, &bin->mtx);
}
}
void
sec_postfork_child(tsdn_t *tsdn, sec_t *sec) {
for (size_t i = 0; i < sec->opts.nshards; i++) {
malloc_mutex_postfork_child(tsdn, &sec->shards[i].mtx);
if (!sec_is_used(sec)) {
return;
}
size_t ntotal_bins = sec->opts.nshards * sec->npsizes;
for (pszind_t i = 0; i < ntotal_bins; i++) {
sec_bin_t *bin = &sec->bins[i];
malloc_mutex_postfork_child(tsdn, &bin->mtx);
}
}

27
src/stats.c
View file

@ -791,9 +791,35 @@ stats_arena_extents_print(emitter_t *emitter, unsigned i) {
static void
stats_arena_hpa_shard_sec_print(emitter_t *emitter, unsigned i) {
size_t sec_bytes;
size_t sec_hits;
size_t sec_misses;
size_t sec_dalloc_flush;
size_t sec_dalloc_noflush;
size_t sec_overfills;
CTL_M2_GET("stats.arenas.0.hpa_sec_bytes", i, &sec_bytes, size_t);
emitter_kv(emitter, "sec_bytes", "Bytes in small extent cache",
emitter_type_size, &sec_bytes);
CTL_M2_GET("stats.arenas.0.hpa_sec_hits", i, &sec_hits, size_t);
emitter_kv(emitter, "sec_hits", "Total hits in small extent cache",
emitter_type_size, &sec_hits);
CTL_M2_GET("stats.arenas.0.hpa_sec_misses", i, &sec_misses, size_t);
emitter_kv(emitter, "sec_misses", "Total misses in small extent cache",
emitter_type_size, &sec_misses);
CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_noflush", i,
&sec_dalloc_noflush, size_t);
emitter_kv(emitter, "sec_dalloc_noflush",
"Dalloc calls without flush in small extent cache",
emitter_type_size, &sec_dalloc_noflush);
CTL_M2_GET("stats.arenas.0.hpa_sec_dalloc_flush", i, &sec_dalloc_flush,
size_t);
emitter_kv(emitter, "sec_dalloc_flush",
"Dalloc calls with flush in small extent cache", emitter_type_size,
&sec_dalloc_flush);
CTL_M2_GET(
"stats.arenas.0.hpa_sec_overfills", i, &sec_overfills, size_t);
emitter_kv(emitter, "sec_overfills",
"sec_fill calls that went over max_bytes", emitter_type_size,
&sec_overfills);
}
static void
@ -1642,7 +1668,6 @@ stats_general_print(emitter_t *emitter) {
OPT_WRITE_SIZE_T("hpa_sec_nshards")
OPT_WRITE_SIZE_T("hpa_sec_max_alloc")
OPT_WRITE_SIZE_T("hpa_sec_max_bytes")
OPT_WRITE_SIZE_T("hpa_sec_bytes_after_flush")
OPT_WRITE_SIZE_T("hpa_sec_batch_fill_extra")
OPT_WRITE_BOOL("huge_arena_pac_thp")
OPT_WRITE_CHAR_P("metadata_thp")

88
test/unit/hpa.c
View file

@ -113,10 +113,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
err = hpa_central_init(&test_data->central, test_data->base, hooks);
assert_false(err, "");
err = hpa_shard_init(&test_data->shard, &test_data->central,
sec_opts_t sec_opts;
sec_opts.nshards = 0;
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
&test_data->emap, test_data->base, &test_data->shard_edata_cache,
SHARD_IND, opts);
SHARD_IND, opts, &sec_opts);
assert_false(err, "");
return (hpa_shard_t *)test_data;
@ -309,83 +311,6 @@ TEST_BEGIN(test_stress) {
}
TEST_END
static void
expect_contiguous(edata_t **edatas, size_t nedatas) {
for (size_t i = 0; i < nedatas; i++) {
size_t expected = (size_t)edata_base_get(edatas[0]) + i * PAGE;
expect_zu_eq(expected, (size_t)edata_base_get(edatas[i]),
"Mismatch at index %zu", i);
}
}
TEST_BEGIN(test_alloc_dalloc_batch) {
test_skip_if(!hpa_supported());
hpa_shard_t *shard = create_test_data(
&hpa_hooks_default, &test_hpa_shard_opts_default);
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
bool deferred_work_generated = false;
enum { NALLOCS = 8 };
edata_t *allocs[NALLOCS];
/*
* Allocate a mix of ways; first half from regular alloc, second half
* from alloc_batch.
*/
for (size_t i = 0; i < NALLOCS / 2; i++) {
allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE,
/* zero */ false, /* guarded */ false,
/* frequent_reuse */ false, &deferred_work_generated);
expect_ptr_not_null(allocs[i], "Unexpected alloc failure");
}
edata_list_active_t allocs_list;
edata_list_active_init(&allocs_list);
size_t nsuccess = pai_alloc_batch(tsdn, &shard->pai, PAGE, NALLOCS / 2,
&allocs_list, /* frequent_reuse */ false, &deferred_work_generated);
expect_zu_eq(NALLOCS / 2, nsuccess, "Unexpected oom");
for (size_t i = NALLOCS / 2; i < NALLOCS; i++) {
allocs[i] = edata_list_active_first(&allocs_list);
edata_list_active_remove(&allocs_list, allocs[i]);
}
/*
* Should have allocated them contiguously, despite the differing
* methods used.
*/
void *orig_base = edata_base_get(allocs[0]);
expect_contiguous(allocs, NALLOCS);
/*
* Batch dalloc the first half, individually deallocate the second half.
*/
for (size_t i = 0; i < NALLOCS / 2; i++) {
edata_list_active_append(&allocs_list, allocs[i]);
}
pai_dalloc_batch(
tsdn, &shard->pai, &allocs_list, &deferred_work_generated);
for (size_t i = NALLOCS / 2; i < NALLOCS; i++) {
pai_dalloc(
tsdn, &shard->pai, allocs[i], &deferred_work_generated);
}
/* Reallocate (individually), and ensure reuse and contiguity. */
for (size_t i = 0; i < NALLOCS; i++) {
allocs[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE,
/* zero */ false, /* guarded */ false, /* frequent_reuse */
false, &deferred_work_generated);
expect_ptr_not_null(allocs[i], "Unexpected alloc failure.");
}
void *new_base = edata_base_get(allocs[0]);
expect_ptr_eq(
orig_base, new_base, "Failed to reuse the allocated memory.");
expect_contiguous(allocs, NALLOCS);
destroy_test_data(shard);
}
TEST_END
static uintptr_t defer_bump_ptr = HUGEPAGE * 123;
static void *
defer_test_map(size_t size) {
@ -1533,8 +1458,7 @@ main(void) {
(void)mem_tree_iter;
(void)mem_tree_reverse_iter;
(void)mem_tree_destroy;
return test_no_reentrancy(test_alloc_max, test_stress,
test_alloc_dalloc_batch, test_defer_time,
return test_no_reentrancy(test_alloc_max, test_stress, test_defer_time,
test_purge_no_infinite_loop, test_no_min_purge_interval,
test_min_purge_interval, test_purge,
test_experimental_max_purge_nhp, test_vectorized_opt_eq_zero,

239
test/unit/hpa_sec_integration.c Normal file
View file

@ -0,0 +1,239 @@
#include "test/jemalloc_test.h"
#include "jemalloc/internal/hpa.h"
#include "jemalloc/internal/nstime.h"
#define SHARD_IND 111
#define ALLOC_MAX (HUGEPAGE)
typedef struct test_data_s test_data_t;
struct test_data_s {
/*
* Must be the first member -- we convert back and forth between the
* test_data_t and the hpa_shard_t;
*/
hpa_shard_t shard;
hpa_central_t central;
base_t *base;
edata_cache_t shard_edata_cache;
emap_t emap;
};
static hpa_shard_opts_t test_hpa_shard_opts = {
/* slab_max_alloc */
HUGEPAGE,
/* hugification_threshold */
0.9 * HUGEPAGE,
/* dirty_mult */
FXP_INIT_PERCENT(10),
/* deferral_allowed */
true,
/* hugify_delay_ms */
0,
/* hugify_sync */
false,
/* min_purge_interval_ms */
5,
/* experimental_max_purge_nhp */
-1,
/* purge_threshold */
PAGE,
/* min_purge_delay_ms */
10,
/* hugify_style */
hpa_hugify_style_lazy};
static hpa_shard_t *
create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts,
const sec_opts_t *sec_opts) {
bool err;
base_t *base = base_new(TSDN_NULL, /* ind */ SHARD_IND,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(base, "");
test_data_t *test_data = malloc(sizeof(test_data_t));
assert_ptr_not_null(test_data, "");
test_data->base = base;
err = edata_cache_init(&test_data->shard_edata_cache, base);
assert_false(err, "");
err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false);
assert_false(err, "");
err = hpa_central_init(&test_data->central, test_data->base, hooks);
assert_false(err, "");
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
&test_data->emap, test_data->base, &test_data->shard_edata_cache,
SHARD_IND, opts, sec_opts);
assert_false(err, "");
return (hpa_shard_t *)test_data;
}
static void
destroy_test_data(hpa_shard_t *shard) {
test_data_t *test_data = (test_data_t *)shard;
base_delete(TSDN_NULL, test_data->base);
free(test_data);
}
static uintptr_t defer_bump_ptr = HUGEPAGE * 123;
static void *
defer_test_map(size_t size) {
void *result = (void *)defer_bump_ptr;
defer_bump_ptr += size;
return result;
}
static void
defer_test_unmap(void *ptr, size_t size) {
(void)ptr;
(void)size;
}
static size_t ndefer_purge_calls = 0;
static size_t npurge_size = 0;
static void
defer_test_purge(void *ptr, size_t size) {
(void)ptr;
npurge_size = size;
++ndefer_purge_calls;
}
static bool defer_vectorized_purge_called = false;
static bool
defer_vectorized_purge(void *vec, size_t vlen, size_t nbytes) {
(void)vec;
(void)nbytes;
++ndefer_purge_calls;
defer_vectorized_purge_called = true;
return false;
}
static size_t ndefer_hugify_calls = 0;
static bool
defer_test_hugify(void *ptr, size_t size, bool sync) {
++ndefer_hugify_calls;
return false;
}
static size_t ndefer_dehugify_calls = 0;
static void
defer_test_dehugify(void *ptr, size_t size) {
++ndefer_dehugify_calls;
}
static nstime_t defer_curtime;
static void
defer_test_curtime(nstime_t *r_time, bool first_reading) {
*r_time = defer_curtime;
}
static uint64_t
defer_test_ms_since(nstime_t *past_time) {
return (nstime_ns(&defer_curtime) - nstime_ns(past_time)) / 1000 / 1000;
}
// test that freed pages stay in SEC and hpa thinks they are active
TEST_BEGIN(test_hpa_sec) {
test_skip_if(!hpa_supported());
hpa_hooks_t hooks;
hooks.map = &defer_test_map;
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
hpa_shard_opts_t opts = test_hpa_shard_opts;
enum { NALLOCS = 8 };
sec_opts_t sec_opts;
sec_opts.nshards = 1;
sec_opts.max_alloc = 2 * PAGE;
sec_opts.max_bytes = NALLOCS * PAGE;
sec_opts.batch_fill_extra = 4;
hpa_shard_t *shard = create_test_data(&hooks, &opts, &sec_opts);
bool deferred_work_generated = false;
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
/* alloc 1 PAGE, confirm sec has fill_extra bytes. */
edata_t *edata1 = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false,
false, &deferred_work_generated);
expect_ptr_not_null(edata1, "Unexpected null edata");
hpa_shard_stats_t hpa_stats;
memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
expect_zu_eq(hpa_stats.psset_stats.merged.nactive,
1 + sec_opts.batch_fill_extra, "");
expect_zu_eq(hpa_stats.secstats.bytes, PAGE * sec_opts.batch_fill_extra,
"sec should have fill extra pages");
/* Alloc/dealloc NALLOCS times and confirm extents are in sec. */
edata_t *edatas[NALLOCS];
for (int i = 0; i < NALLOCS; i++) {
edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false,
false, false, &deferred_work_generated);
expect_ptr_not_null(edatas[i], "Unexpected null edata");
}
memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
expect_zu_eq(hpa_stats.psset_stats.merged.nactive, 2 + NALLOCS, "");
expect_zu_eq(hpa_stats.secstats.bytes, PAGE, "2 refills (at 0 and 4)");
for (int i = 0; i < NALLOCS - 1; i++) {
pai_dalloc(
tsdn, &shard->pai, edatas[i], &deferred_work_generated);
}
memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
expect_zu_eq(hpa_stats.psset_stats.merged.nactive, (2 + NALLOCS), "");
expect_zu_eq(
hpa_stats.secstats.bytes, sec_opts.max_bytes, "sec should be full");
/* this one should flush 1 + 0.25 * 8 = 3 extents */
pai_dalloc(
tsdn, &shard->pai, edatas[NALLOCS - 1], &deferred_work_generated);
memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
expect_zu_eq(hpa_stats.psset_stats.merged.nactive, (NALLOCS - 1), "");
expect_zu_eq(hpa_stats.psset_stats.merged.ndirty, 3, "");
expect_zu_eq(hpa_stats.secstats.bytes, 0.75 * sec_opts.max_bytes,
"sec should be full");
/* Next allocation should come from SEC and not increase active */
edata_t *edata2 = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, false,
false, &deferred_work_generated);
expect_ptr_not_null(edata2, "Unexpected null edata");
memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
expect_zu_eq(hpa_stats.psset_stats.merged.nactive, NALLOCS - 1, "");
expect_zu_eq(hpa_stats.secstats.bytes, 0.75 * sec_opts.max_bytes - PAGE,
"sec should have max_bytes minus one page that just came from it");
/* We return this one and it stays in the cache */
pai_dalloc(tsdn, &shard->pai, edata2, &deferred_work_generated);
memset(&hpa_stats, 0, sizeof(hpa_shard_stats_t));
hpa_shard_stats_merge(tsdn, shard, &hpa_stats);
expect_zu_eq(hpa_stats.psset_stats.merged.nactive, NALLOCS - 1, "");
expect_zu_eq(hpa_stats.psset_stats.merged.ndirty, 3, "");
expect_zu_eq(hpa_stats.secstats.bytes, 0.75 * sec_opts.max_bytes, "");
destroy_test_data(shard);
}
TEST_END
int
main(void) {
return test_no_reentrancy(test_hpa_sec);
}

3
test/unit/hpa_sec_integration.sh Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
export MALLOC_CONF="process_madvise_max_batch:0,experimental_hpa_start_huge_if_thp_always:false"

8
test/unit/hpa_thp_always.c
View file

@ -65,10 +65,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
err = hpa_central_init(&test_data->central, test_data->base, hooks);
assert_false(err, "");
err = hpa_shard_init(&test_data->shard, &test_data->central,
sec_opts_t sec_opts;
sec_opts.nshards = 0;
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
&test_data->emap, test_data->base, &test_data->shard_edata_cache,
SHARD_IND, opts);
SHARD_IND, opts, &sec_opts);
assert_false(err, "");
return (hpa_shard_t *)test_data;

7
test/unit/hpa_vectorized_madvise.c
View file

@ -66,9 +66,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
err = hpa_central_init(&test_data->central, test_data->base, hooks);
assert_false(err, "");
err = hpa_shard_init(&test_data->shard, &test_data->central,
sec_opts_t sec_opts;
sec_opts.nshards = 0;
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
&test_data->emap, test_data->base, &test_data->shard_edata_cache,
SHARD_IND, opts);
SHARD_IND, opts, &sec_opts);
assert_false(err, "");
return (hpa_shard_t *)test_data;

8
test/unit/hpa_vectorized_madvise_large_batch.c
View file

@ -66,10 +66,12 @@ create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
err = hpa_central_init(&test_data->central, test_data->base, hooks);
assert_false(err, "");
err = hpa_shard_init(&test_data->shard, &test_data->central,
sec_opts_t sec_opts;
sec_opts.nshards = 0;
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
err = hpa_shard_init(tsdn, &test_data->shard, &test_data->central,
&test_data->emap, test_data->base, &test_data->shard_edata_cache,
SHARD_IND, opts);
SHARD_IND, opts, &sec_opts);
assert_false(err, "");
return (hpa_shard_t *)test_data;

1
test/unit/mallctl.c
View file

@ -313,7 +313,6 @@ TEST_BEGIN(test_mallctl_opt) {
TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always);
TEST_MALLCTL_OPT(size_t, hpa_sec_max_alloc, always);
TEST_MALLCTL_OPT(size_t, hpa_sec_max_bytes, always);
TEST_MALLCTL_OPT(size_t, hpa_sec_bytes_after_flush, always);
TEST_MALLCTL_OPT(size_t, hpa_sec_batch_fill_extra, always);
TEST_MALLCTL_OPT(ssize_t, experimental_hpa_max_purge_nhp, always);
TEST_MALLCTL_OPT(size_t, hpa_purge_threshold, always);

1043
test/unit/sec.c
View file

File diff suppressed because it is too large Load diff