mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-07-19 23:27:18 +03:00
Use SEC in PAC to reduce lock contention on the ecaches
Add a small extent cache in front of the PAC ecaches. Allocs and dallocs that fit are served from per-shard SEC bins without taking the ecache mutex; overflow falls through to the backing ecaches, including ecache_pinned for pinned extents. The feature is gated behind experimental_pac_sec_nshards (default 0, disabled). To support independent HPA and PAC SEC instances, sec_alloc/sec_dalloc/sec_fill take an explicit shard argument, with HPA and PAC using separate TSD shard slots.
This commit is contained in:
Bin Liu
parent
8aa7e5ec03
commit
ccf9cdf417
19 changed files with 986 additions and 102 deletions
|
|
@ -181,7 +181,7 @@ static const size_t num_global_mutexes = sizeof(global_mutex_names)
|
|||
static const char *arena_mutex_names[] = {"large", "extent_avail",
|
||||
"extents_dirty", "extents_muzzy", "extents_retained", "decay_dirty",
|
||||
"decay_muzzy", "base", "tcache_list", "hpa_shard", "hpa_shard_grow",
|
||||
"hpa_sec"};
|
||||
"hpa_sec", "pac_sec"};
|
||||
static const size_t num_arena_mutexes = sizeof(arena_mutex_names)
|
||||
/ sizeof(arena_mutex_names[0]);
|
||||
|
||||
|
|
|
|||
|
|
@ -530,6 +530,9 @@ TEST_BEGIN(test_mallctl_opt) {
|
|||
TEST_MALLCTL_OPT(size_t, hpa_sec_nshards, always);
|
||||
TEST_MALLCTL_OPT(size_t, hpa_sec_max_alloc, always);
|
||||
TEST_MALLCTL_OPT(size_t, hpa_sec_max_bytes, always);
|
||||
TEST_MALLCTL_OPT(size_t, experimental_pac_sec_nshards, always);
|
||||
TEST_MALLCTL_OPT(size_t, experimental_pac_sec_max_alloc, always);
|
||||
TEST_MALLCTL_OPT(size_t, experimental_pac_sec_max_bytes, always);
|
||||
TEST_MALLCTL_OPT(ssize_t, experimental_hpa_max_purge_nhp, always);
|
||||
TEST_MALLCTL_OPT(size_t, hpa_purge_threshold, always);
|
||||
TEST_MALLCTL_OPT(uint64_t, hpa_min_purge_delay_ms, always);
|
||||
|
|
|
|||
387
test/unit/pac_sec_integration.c
Normal file
387
test/unit/pac_sec_integration.c
Normal file
|
|
@ -0,0 +1,387 @@
|
|||
#include "test/jemalloc_test.h"
|
||||
|
||||
/*
|
||||
* Use multiple shards while keeping the single-threaded shard choice
|
||||
* deterministic. Disable background threads to avoid decay races.
|
||||
*/
|
||||
const char *malloc_conf =
|
||||
"experimental_pac_sec_nshards:2,background_thread:false";
|
||||
|
||||
static sec_opts_t saved_pac_sec_opts;
|
||||
|
||||
static void
|
||||
pac_sec_test_opts_set(void) {
|
||||
saved_pac_sec_opts = opt_pac_sec_opts;
|
||||
/*
|
||||
* Include cache-oblivious padding so PAC can cache the test size.
|
||||
*/
|
||||
size_t test_extent_size = SC_LARGE_MINCLASS + sz_large_pad;
|
||||
opt_pac_sec_opts.max_alloc = test_extent_size;
|
||||
opt_pac_sec_opts.max_bytes = 4 * test_extent_size;
|
||||
}
|
||||
|
||||
static void
|
||||
pac_sec_test_opts_restore(void) {
|
||||
opt_pac_sec_opts = saved_pac_sec_opts;
|
||||
}
|
||||
|
||||
static void *
|
||||
pinned_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr,
|
||||
size_t size, size_t alignment, bool *zero, bool *commit,
|
||||
unsigned arena_ind) {
|
||||
void *ret = ehooks_default_extent_hooks.alloc(
|
||||
(extent_hooks_t *)&ehooks_default_extent_hooks, new_addr, size,
|
||||
alignment, zero, commit, arena_ind);
|
||||
if (ret == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
if (!*commit) {
|
||||
if (ehooks_default_extent_hooks.commit != NULL
|
||||
&& ehooks_default_extent_hooks.commit(
|
||||
(extent_hooks_t *)&ehooks_default_extent_hooks, ret,
|
||||
size, 0, size, arena_ind)) {
|
||||
ehooks_default_extent_hooks.dalloc(
|
||||
(extent_hooks_t *)&ehooks_default_extent_hooks, ret,
|
||||
size, *commit, arena_ind);
|
||||
return NULL;
|
||||
}
|
||||
*commit = true;
|
||||
}
|
||||
return (void *)((uintptr_t)ret | EXTENT_ALLOC_FLAG_PINNED);
|
||||
}
|
||||
|
||||
static void
|
||||
pinned_extent_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
bool committed, unsigned arena_ind) {
|
||||
ehooks_default_extent_hooks.destroy(
|
||||
(extent_hooks_t *)&ehooks_default_extent_hooks, addr, size,
|
||||
committed, arena_ind);
|
||||
}
|
||||
|
||||
static bool
|
||||
pinned_extent_split(extent_hooks_t *extent_hooks, void *addr, size_t size,
|
||||
size_t size_a, size_t size_b, bool committed, unsigned arena_ind) {
|
||||
return ehooks_default_extent_hooks.split(
|
||||
(extent_hooks_t *)&ehooks_default_extent_hooks, addr, size, size_a,
|
||||
size_b, committed, arena_ind);
|
||||
}
|
||||
|
||||
static bool
|
||||
pinned_extent_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a,
|
||||
void *addr_b, size_t size_b, bool committed, unsigned arena_ind) {
|
||||
return ehooks_default_extent_hooks.merge(
|
||||
(extent_hooks_t *)&ehooks_default_extent_hooks, addr_a, size_a,
|
||||
addr_b, size_b, committed, arena_ind);
|
||||
}
|
||||
|
||||
static extent_hooks_t pinned_hooks = {
|
||||
pinned_extent_alloc,
|
||||
NULL, /* dalloc */
|
||||
pinned_extent_destroy,
|
||||
NULL, /* commit */
|
||||
NULL, /* decommit */
|
||||
NULL, /* purge_lazy */
|
||||
NULL, /* purge_forced */
|
||||
pinned_extent_split,
|
||||
pinned_extent_merge
|
||||
};
|
||||
|
||||
static size_t
|
||||
read_arena_stat(unsigned arena_ind, const char *field) {
|
||||
char cmd[128];
|
||||
size_t val;
|
||||
size_t sz = sizeof(val);
|
||||
uint64_t epoch = 1;
|
||||
sz = sizeof(epoch);
|
||||
expect_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sz), 0,
|
||||
"Unexpected mallctl failure");
|
||||
sz = sizeof(val);
|
||||
snprintf(cmd, sizeof(cmd), "stats.arenas.%u.%s", arena_ind, field);
|
||||
expect_d_eq(mallctl(cmd, (void *)&val, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure reading arena stat");
|
||||
return val;
|
||||
}
|
||||
|
||||
static size_t
|
||||
read_stat(unsigned arena_ind, const char *field) {
|
||||
char cmd[128];
|
||||
snprintf(cmd, sizeof(cmd), "pac_sec_%s", field);
|
||||
return read_arena_stat(arena_ind, cmd);
|
||||
}
|
||||
|
||||
static size_t
|
||||
read_extents_stat(unsigned arena_ind, pszind_t pszind, const char *field) {
|
||||
char cmd[128];
|
||||
snprintf(cmd, sizeof(cmd), "extents.%u.%s", pszind, field);
|
||||
return read_arena_stat(arena_ind, cmd);
|
||||
}
|
||||
|
||||
static size_t
|
||||
read_pinned_npages(unsigned arena_ind) {
|
||||
tsd_t *tsd = tsd_fetch();
|
||||
arena_t *arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
|
||||
expect_ptr_not_null(arena, "arena_get failed");
|
||||
return ecache_npages_get(&arena->pa_shard.pac.ecache_pinned);
|
||||
}
|
||||
|
||||
static void
|
||||
dirty_decay_ms_set(unsigned arena_ind, ssize_t decay_ms) {
|
||||
char cmd[64];
|
||||
snprintf(cmd, sizeof(cmd), "arena.%u.dirty_decay_ms", arena_ind);
|
||||
expect_d_eq(mallctl(cmd, NULL, NULL, (void *)&decay_ms,
|
||||
sizeof(decay_ms)), 0, "dirty_decay_ms mallctl failed");
|
||||
}
|
||||
|
||||
TEST_BEGIN(test_pac_sec_alloc_dalloc_cycle) {
|
||||
test_skip_if(!config_stats);
|
||||
test_skip_if(opt_hpa);
|
||||
|
||||
pac_sec_test_opts_set();
|
||||
unsigned arena_ind;
|
||||
size_t sz = sizeof(arena_ind);
|
||||
expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
|
||||
0, "Unexpected arenas.create failure");
|
||||
|
||||
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
|
||||
size_t alloc_size = SC_LARGE_MINCLASS;
|
||||
|
||||
size_t max_bytes;
|
||||
sz = sizeof(max_bytes);
|
||||
expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes",
|
||||
(void *)&max_bytes, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
|
||||
size_t capacity = max_bytes / alloc_size;
|
||||
expect_zu_gt(capacity, 0, "SEC capacity must be > 0 for this test");
|
||||
|
||||
/* Initial allocation misses SEC. */
|
||||
void *p1 = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p1, "mallocx failed");
|
||||
size_t resident_after_alloc = read_arena_stat(arena_ind, "resident");
|
||||
expect_zu_eq(read_stat(arena_ind, "misses"), 1,
|
||||
"first alloc should miss SEC");
|
||||
expect_zu_eq(read_stat(arena_ind, "hits"), 0,
|
||||
"no hits yet");
|
||||
expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
|
||||
"SEC should be empty (extent is active)");
|
||||
|
||||
/* Freeing the extent caches it in PAC SEC. */
|
||||
dallocx(p1, flags);
|
||||
size_t cached_after_one = read_stat(arena_ind, "bytes");
|
||||
expect_zu_gt(cached_after_one, 0,
|
||||
"SEC should cache the freed extent");
|
||||
expect_zu_ge(read_arena_stat(arena_ind, "pdirty"),
|
||||
cached_after_one >> LG_PAGE,
|
||||
"SEC bytes should count toward dirty page stats");
|
||||
expect_zu_ge(read_arena_stat(arena_ind, "resident"),
|
||||
resident_after_alloc,
|
||||
"SEC bytes should remain included in resident stats");
|
||||
/* Use the actual extent size, including size class rounding. */
|
||||
size_t extent_size = cached_after_one;
|
||||
pszind_t extent_pszind = sz_psz2ind(extent_size);
|
||||
expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "ndirty"), 1,
|
||||
"SEC extents should count toward extents dirty stats");
|
||||
expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "dirty_bytes"),
|
||||
cached_after_one,
|
||||
"SEC bytes should count toward extents dirty byte stats");
|
||||
expect_zu_eq(read_stat(arena_ind, "dalloc_noflush"), 1,
|
||||
"one dalloc absorbed without flush");
|
||||
expect_zu_eq(read_stat(arena_ind, "dalloc_flush"), 0,
|
||||
"no flush yet");
|
||||
|
||||
/* Recompute capacity based on actual extent size. */
|
||||
capacity = max_bytes / extent_size;
|
||||
expect_zu_gt(capacity, 0, "SEC capacity should be positive");
|
||||
|
||||
/* Reallocate from SEC. */
|
||||
void *p2 = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p2, "mallocx failed");
|
||||
expect_zu_eq(read_stat(arena_ind, "hits"), 1,
|
||||
"second alloc should hit SEC");
|
||||
expect_zu_eq(read_stat(arena_ind, "misses"), 1,
|
||||
"misses should not increase");
|
||||
expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
|
||||
"SEC should be empty after hit");
|
||||
|
||||
dallocx(p2, flags);
|
||||
|
||||
/* Overflow SEC and flush cold extents to ecache_dirty. */
|
||||
size_t nallocs = capacity + 2;
|
||||
void **ptrs = mallocx(nallocs * sizeof(void *),
|
||||
MALLOCX_TCACHE_NONE);
|
||||
expect_ptr_not_null(ptrs, "metadata alloc failed");
|
||||
|
||||
for (size_t i = 0; i < nallocs; i++) {
|
||||
ptrs[i] = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i);
|
||||
}
|
||||
for (size_t i = 0; i < nallocs; i++) {
|
||||
dallocx(ptrs[i], flags);
|
||||
}
|
||||
|
||||
size_t noflush = read_stat(arena_ind, "dalloc_noflush");
|
||||
size_t flush = read_stat(arena_ind, "dalloc_flush");
|
||||
size_t cached_bytes = read_stat(arena_ind, "bytes");
|
||||
|
||||
expect_zu_gt(noflush, 1,
|
||||
"most dallocs should be absorbed");
|
||||
expect_zu_gt(flush, 0,
|
||||
"overflow should trigger at least one flush");
|
||||
expect_zu_gt(cached_bytes, 0,
|
||||
"SEC should still hold extents after partial flush");
|
||||
expect_zu_le(cached_bytes, max_bytes,
|
||||
"SEC should not exceed max_bytes");
|
||||
|
||||
/* A populated SEC serves the next allocation. */
|
||||
size_t misses_before = read_stat(arena_ind, "misses");
|
||||
void *p3 = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p3, "mallocx failed");
|
||||
expect_zu_eq(read_stat(arena_ind, "misses"), misses_before,
|
||||
"alloc from populated SEC should not miss");
|
||||
dallocx(p3, flags);
|
||||
|
||||
/* Purge flushes SEC entirely. */
|
||||
char cmd[64];
|
||||
snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind);
|
||||
expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
|
||||
"purge failed");
|
||||
expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
|
||||
"SEC should be empty after purge");
|
||||
|
||||
/* Allocation after purge misses SEC. */
|
||||
size_t hits_before = read_stat(arena_ind, "hits");
|
||||
void *p4 = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p4, "mallocx failed");
|
||||
expect_zu_eq(read_stat(arena_ind, "hits"), hits_before,
|
||||
"alloc after purge should miss SEC");
|
||||
dallocx(p4, flags);
|
||||
|
||||
dallocx(ptrs, MALLOCX_TCACHE_NONE);
|
||||
snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind);
|
||||
expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
|
||||
"arena destroy failed");
|
||||
pac_sec_test_opts_restore();
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_pac_sec_dirty_decay_toggle) {
|
||||
test_skip_if(!config_stats);
|
||||
test_skip_if(opt_hpa);
|
||||
|
||||
pac_sec_test_opts_set();
|
||||
unsigned arena_ind;
|
||||
size_t sz = sizeof(arena_ind);
|
||||
expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz, NULL, 0),
|
||||
0, "Unexpected arenas.create failure");
|
||||
|
||||
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
|
||||
size_t alloc_size = SC_LARGE_MINCLASS;
|
||||
|
||||
void *p = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p, "mallocx failed");
|
||||
dallocx(p, flags);
|
||||
expect_zu_gt(read_stat(arena_ind, "bytes"), 0,
|
||||
"SEC should cache when dirty decay is enabled");
|
||||
|
||||
dirty_decay_ms_set(arena_ind, 0);
|
||||
expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
|
||||
"disabling dirty decay should flush SEC");
|
||||
|
||||
p = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p, "mallocx failed");
|
||||
dallocx(p, flags);
|
||||
expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
|
||||
"SEC should stay disabled while dirty decay is zero");
|
||||
|
||||
dirty_decay_ms_set(arena_ind, 100);
|
||||
p = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p, "mallocx failed");
|
||||
dallocx(p, flags);
|
||||
expect_zu_gt(read_stat(arena_ind, "bytes"), 0,
|
||||
"SEC should be usable after dirty decay is re-enabled");
|
||||
|
||||
char cmd[64];
|
||||
snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind);
|
||||
expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
|
||||
"arena destroy failed");
|
||||
pac_sec_test_opts_restore();
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_pac_sec_flush_pinned) {
|
||||
test_skip_if(!config_stats);
|
||||
test_skip_if(opt_hpa);
|
||||
|
||||
pac_sec_test_opts_set();
|
||||
unsigned arena_ind;
|
||||
size_t sz = sizeof(arena_ind);
|
||||
extent_hooks_t *hooks_ptr = &pinned_hooks;
|
||||
expect_d_eq(mallctl("arenas.create", (void *)&arena_ind, &sz,
|
||||
&hooks_ptr, sizeof(hooks_ptr)), 0,
|
||||
"Unexpected arenas.create failure");
|
||||
|
||||
int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE;
|
||||
size_t alloc_size = SC_LARGE_MINCLASS;
|
||||
size_t max_bytes;
|
||||
sz = sizeof(max_bytes);
|
||||
expect_d_eq(mallctl("opt.experimental_pac_sec_max_bytes",
|
||||
(void *)&max_bytes, &sz, NULL, 0), 0,
|
||||
"Unexpected mallctl failure");
|
||||
|
||||
void *p = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(p, "mallocx failed");
|
||||
size_t resident_after_alloc = read_arena_stat(arena_ind, "resident");
|
||||
dallocx(p, flags);
|
||||
size_t sec_bytes = read_stat(arena_ind, "bytes");
|
||||
expect_zu_gt(sec_bytes, 0, "SEC should cache the pinned extent");
|
||||
expect_zu_ge(read_arena_stat(arena_ind, "pinned"), sec_bytes,
|
||||
"Pinned bytes cached in SEC should count toward pinned stats");
|
||||
expect_zu_ge(read_arena_stat(arena_ind, "resident"),
|
||||
resident_after_alloc,
|
||||
"Pinned SEC bytes should remain included in resident stats");
|
||||
|
||||
size_t extent_size = sec_bytes;
|
||||
pszind_t extent_pszind = sz_psz2ind(extent_size);
|
||||
expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "npinned"), 1,
|
||||
"Pinned SEC extents should count toward extents pinned stats");
|
||||
expect_zu_ge(read_extents_stat(arena_ind, extent_pszind, "pinned_bytes"),
|
||||
sec_bytes,
|
||||
"Pinned SEC bytes should count toward extents pinned byte stats");
|
||||
size_t nallocs = max_bytes / extent_size + 2;
|
||||
void **ptrs = mallocx(nallocs * sizeof(void *), MALLOCX_TCACHE_NONE);
|
||||
expect_ptr_not_null(ptrs, "metadata alloc failed");
|
||||
for (size_t i = 0; i < nallocs; i++) {
|
||||
ptrs[i] = mallocx(alloc_size, flags);
|
||||
expect_ptr_not_null(ptrs[i], "mallocx %zu failed", i);
|
||||
}
|
||||
size_t pinned_before_overflow = read_pinned_npages(arena_ind);
|
||||
for (size_t i = 0; i < nallocs; i++) {
|
||||
dallocx(ptrs[i], flags);
|
||||
}
|
||||
expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_overflow,
|
||||
"SEC overflow should flush pinned extents to ecache_pinned");
|
||||
|
||||
size_t pinned_before_purge = read_pinned_npages(arena_ind);
|
||||
char cmd[64];
|
||||
snprintf(cmd, sizeof(cmd), "arena.%u.purge", arena_ind);
|
||||
expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
|
||||
"purge failed");
|
||||
expect_zu_eq(read_stat(arena_ind, "bytes"), 0,
|
||||
"SEC should be empty after purge");
|
||||
expect_zu_gt(read_pinned_npages(arena_ind), pinned_before_purge,
|
||||
"PAC SEC purge should flush pinned extents to ecache_pinned");
|
||||
|
||||
dallocx(ptrs, MALLOCX_TCACHE_NONE);
|
||||
snprintf(cmd, sizeof(cmd), "arena.%u.destroy", arena_ind);
|
||||
expect_d_eq(mallctl(cmd, NULL, NULL, NULL, 0), 0,
|
||||
"arena destroy failed");
|
||||
pac_sec_test_opts_restore();
|
||||
}
|
||||
TEST_END
|
||||
|
||||
int
|
||||
main(void) {
|
||||
return test_no_reentrancy(
|
||||
test_pac_sec_alloc_dalloc_cycle, test_pac_sec_dirty_decay_toggle,
|
||||
test_pac_sec_flush_pinned);
|
||||
}
|
||||
188
test/unit/sec.c
188
test/unit/sec.c
|
|
@ -12,6 +12,17 @@ struct test_data_s {
|
|||
base_t *base;
|
||||
};
|
||||
|
||||
static void
|
||||
edata_init_test(edata_t *edata) {
|
||||
memset(edata, 0, sizeof(*edata));
|
||||
}
|
||||
|
||||
static void
|
||||
edata_init_test_size(edata_t *edata, size_t size) {
|
||||
edata_init_test(edata);
|
||||
edata_size_set(edata, size);
|
||||
}
|
||||
|
||||
static void
|
||||
test_data_init(tsdn_t *tsdn, test_data_t *tdata, const sec_opts_t *opts) {
|
||||
tdata->base = base_new(TSDN_NULL, /* ind */ 123,
|
||||
|
|
@ -31,6 +42,32 @@ destroy_test_data(tsdn_t *tsdn, test_data_t *tdata) {
|
|||
base_delete(tsdn, tdata->base);
|
||||
}
|
||||
|
||||
static uint8_t
|
||||
test_sec_shard(tsdn_t *tsdn, sec_t *sec) {
|
||||
if (tsdn_null(tsdn) || sec->opts.nshards <= 1) {
|
||||
return 0;
|
||||
}
|
||||
tsd_t *tsd = tsdn_tsd(tsdn);
|
||||
return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd));
|
||||
}
|
||||
|
||||
static edata_t *
|
||||
sec_test_alloc(tsdn_t *tsdn, sec_t *sec, size_t size) {
|
||||
return sec_alloc(tsdn, sec, size, test_sec_shard(tsdn, sec));
|
||||
}
|
||||
|
||||
static void
|
||||
sec_test_fill(tsdn_t *tsdn, sec_t *sec, size_t size,
|
||||
edata_list_active_t *result, size_t nallocs) {
|
||||
sec_fill(tsdn, sec, size, result, nallocs, test_sec_shard(tsdn, sec));
|
||||
}
|
||||
|
||||
static void
|
||||
sec_test_dalloc(tsdn_t *tsdn, sec_t *sec,
|
||||
edata_list_active_t *dalloc_list) {
|
||||
sec_dalloc(tsdn, sec, dalloc_list, test_sec_shard(tsdn, sec));
|
||||
}
|
||||
|
||||
TEST_BEGIN(test_max_nshards_option_zero) {
|
||||
test_data_t tdata;
|
||||
sec_opts_t opts;
|
||||
|
|
@ -41,7 +78,8 @@ TEST_BEGIN(test_max_nshards_option_zero) {
|
|||
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
|
||||
test_data_init(tsdn, &tdata, &opts);
|
||||
|
||||
edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE);
|
||||
edata_t *edata = sec_size_supported(&tdata.sec, PAGE)
|
||||
? sec_test_alloc(tsdn, &tdata.sec, PAGE) : NULL;
|
||||
expect_ptr_null(edata, "SEC should be disabled when nshards==0");
|
||||
destroy_test_data(tsdn, &tdata);
|
||||
}
|
||||
|
|
@ -57,7 +95,8 @@ TEST_BEGIN(test_max_alloc_option_too_small) {
|
|||
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
|
||||
test_data_init(tsdn, &tdata, &opts);
|
||||
|
||||
edata_t *edata = sec_alloc(tsdn, &tdata.sec, 3 * PAGE);
|
||||
edata_t *edata = sec_size_supported(&tdata.sec, 3 * PAGE)
|
||||
? sec_test_alloc(tsdn, &tdata.sec, 3 * PAGE) : NULL;
|
||||
expect_ptr_null(edata, "max_alloc is 2*PAGE, should not alloc 3*PAGE");
|
||||
destroy_test_data(tsdn, &tdata);
|
||||
}
|
||||
|
|
@ -78,11 +117,11 @@ TEST_BEGIN(test_sec_fill) {
|
|||
edata_list_active_t allocs;
|
||||
edata_list_active_init(&allocs);
|
||||
edata_t edata1, edata2;
|
||||
edata_size_set(&edata1, PAGE);
|
||||
edata_size_set(&edata2, PAGE);
|
||||
edata_init_test_size(&edata1, PAGE);
|
||||
edata_init_test_size(&edata2, PAGE);
|
||||
edata_list_active_append(&allocs, &edata1);
|
||||
edata_list_active_append(&allocs, &edata2);
|
||||
sec_fill(tsdn, &tdata.sec, PAGE, &allocs, 2);
|
||||
sec_test_fill(tsdn, &tdata.sec, PAGE, &allocs, 2);
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
expect_zu_eq(stats.bytes, 2 * PAGE, "SEC should have what we filled");
|
||||
expect_true(edata_list_active_empty(&allocs),
|
||||
|
|
@ -91,13 +130,13 @@ TEST_BEGIN(test_sec_fill) {
|
|||
/* Try to overfill and confirm that max_bytes is respected. */
|
||||
stats.bytes = 0;
|
||||
edata_t edata5, edata4, edata3;
|
||||
edata_size_set(&edata3, PAGE);
|
||||
edata_size_set(&edata4, PAGE);
|
||||
edata_size_set(&edata5, PAGE);
|
||||
edata_init_test_size(&edata3, PAGE);
|
||||
edata_init_test_size(&edata4, PAGE);
|
||||
edata_init_test_size(&edata5, PAGE);
|
||||
edata_list_active_append(&allocs, &edata3);
|
||||
edata_list_active_append(&allocs, &edata4);
|
||||
edata_list_active_append(&allocs, &edata5);
|
||||
sec_fill(tsdn, &tdata.sec, PAGE, &allocs, 3);
|
||||
sec_test_fill(tsdn, &tdata.sec, PAGE, &allocs, 3);
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
expect_zu_eq(
|
||||
stats.bytes, opts.max_bytes, "SEC can't have more than max_bytes");
|
||||
|
|
@ -118,20 +157,20 @@ TEST_BEGIN(test_sec_alloc) {
|
|||
test_data_init(tsdn, &tdata, &opts);
|
||||
|
||||
/* Alloc from empty cache returns NULL */
|
||||
edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE);
|
||||
edata_t *edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
|
||||
expect_ptr_null(edata, "SEC is empty");
|
||||
|
||||
/* Place two extents into the sec */
|
||||
edata_list_active_t allocs;
|
||||
edata_list_active_init(&allocs);
|
||||
edata_t edata1, edata2;
|
||||
edata_size_set(&edata1, PAGE);
|
||||
edata_init_test_size(&edata1, PAGE);
|
||||
edata_list_active_append(&allocs, &edata1);
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_true(edata_list_active_empty(&allocs), "");
|
||||
edata_size_set(&edata2, PAGE);
|
||||
edata_init_test_size(&edata2, PAGE);
|
||||
edata_list_active_append(&allocs, &edata2);
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_true(edata_list_active_empty(&allocs), "");
|
||||
|
||||
sec_stats_t stats = {0};
|
||||
|
|
@ -141,20 +180,20 @@ TEST_BEGIN(test_sec_alloc) {
|
|||
stats.bytes = 0;
|
||||
|
||||
/* Most recently cached extent should be used on alloc */
|
||||
edata = sec_alloc(tsdn, &tdata.sec, PAGE);
|
||||
edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
|
||||
expect_ptr_eq(edata, &edata2, "edata2 is most recently used");
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
expect_zu_eq(stats.bytes, PAGE, "One more item left in the cache");
|
||||
stats.bytes = 0;
|
||||
|
||||
/* Alloc can still get extents from cache */
|
||||
edata = sec_alloc(tsdn, &tdata.sec, PAGE);
|
||||
edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
|
||||
expect_ptr_eq(edata, &edata1, "SEC is not empty");
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
expect_zu_eq(stats.bytes, 0, "No more items after last one is popped");
|
||||
|
||||
/* And cache is empty again */
|
||||
edata = sec_alloc(tsdn, &tdata.sec, PAGE);
|
||||
edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
|
||||
expect_ptr_null(edata, "SEC is empty");
|
||||
destroy_test_data(tsdn, &tdata);
|
||||
}
|
||||
|
|
@ -174,20 +213,20 @@ TEST_BEGIN(test_sec_dalloc) {
|
|||
edata_list_active_t allocs;
|
||||
edata_list_active_init(&allocs);
|
||||
edata_t edata1;
|
||||
edata_size_set(&edata1, PAGE);
|
||||
edata_init_test_size(&edata1, PAGE);
|
||||
edata_list_active_append(&allocs, &edata1);
|
||||
|
||||
/* SEC is empty, we return one pointer to it */
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_true(
|
||||
edata_list_active_empty(&allocs), "extents should be consumed");
|
||||
|
||||
/* Return one more extent, so that we are at the limit */
|
||||
edata_t edata2;
|
||||
edata_size_set(&edata2, PAGE);
|
||||
edata_init_test_size(&edata2, PAGE);
|
||||
edata_list_active_append(&allocs, &edata2);
|
||||
/* Sec can take one more as well and we will be exactly at max_bytes */
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_true(
|
||||
edata_list_active_empty(&allocs), "extents should be consumed");
|
||||
|
||||
|
|
@ -203,9 +242,9 @@ TEST_BEGIN(test_sec_dalloc) {
|
|||
* the one given in the input as it is the most recently used.
|
||||
*/
|
||||
edata_t edata3;
|
||||
edata_size_set(&edata3, PAGE);
|
||||
edata_init_test_size(&edata3, PAGE);
|
||||
edata_list_active_append(&allocs, &edata3);
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_false(
|
||||
edata_list_active_empty(&allocs), "extents should NOT be consumed");
|
||||
expect_ptr_ne(
|
||||
|
|
@ -218,6 +257,65 @@ TEST_BEGIN(test_sec_dalloc) {
|
|||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_sec_pinned_stats) {
|
||||
test_data_t tdata;
|
||||
sec_opts_t opts;
|
||||
opts.nshards = 1;
|
||||
opts.max_alloc = PAGE;
|
||||
opts.max_bytes = 2 * PAGE;
|
||||
|
||||
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
|
||||
test_data_init(tsdn, &tdata, &opts);
|
||||
|
||||
edata_list_active_t allocs;
|
||||
edata_list_active_init(&allocs);
|
||||
edata_t pinned, unpinned;
|
||||
edata_init_test_size(&pinned, PAGE);
|
||||
edata_pinned_set(&pinned, true);
|
||||
edata_init_test_size(&unpinned, PAGE);
|
||||
|
||||
edata_list_active_append(&allocs, &pinned);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
edata_list_active_append(&allocs, &unpinned);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
|
||||
sec_stats_t stats = {0};
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
expect_zu_eq(stats.bytes, 2 * PAGE, "");
|
||||
expect_zu_eq(stats.bytes_pinned, PAGE, "");
|
||||
sec_pszind_stats_t pszind_stats = {0};
|
||||
sec_stats_merge_pszind(
|
||||
tsdn, &tdata.sec, sz_psz2ind(PAGE), &pszind_stats);
|
||||
expect_zu_eq(pszind_stats.nextents, 2, "");
|
||||
expect_zu_eq(pszind_stats.bytes, 2 * PAGE, "");
|
||||
expect_zu_eq(pszind_stats.nextents_pinned, 1, "");
|
||||
expect_zu_eq(pszind_stats.bytes_pinned, PAGE, "");
|
||||
|
||||
memset(&stats, 0, sizeof(stats));
|
||||
expect_ptr_eq(sec_test_alloc(tsdn, &tdata.sec, PAGE), &unpinned,
|
||||
"Unpinned extent is the most recently cached");
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
expect_zu_eq(stats.bytes, PAGE, "");
|
||||
expect_zu_eq(stats.bytes_pinned, PAGE, "");
|
||||
memset(&pszind_stats, 0, sizeof(pszind_stats));
|
||||
sec_stats_merge_pszind(
|
||||
tsdn, &tdata.sec, sz_psz2ind(PAGE), &pszind_stats);
|
||||
expect_zu_eq(pszind_stats.nextents, 1, "");
|
||||
expect_zu_eq(pszind_stats.bytes, PAGE, "");
|
||||
expect_zu_eq(pszind_stats.nextents_pinned, 1, "");
|
||||
expect_zu_eq(pszind_stats.bytes_pinned, PAGE, "");
|
||||
|
||||
memset(&stats, 0, sizeof(stats));
|
||||
expect_ptr_eq(sec_test_alloc(tsdn, &tdata.sec, PAGE), &pinned,
|
||||
"Pinned extent should be tracked until allocated");
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
expect_zu_eq(stats.bytes, 0, "");
|
||||
expect_zu_eq(stats.bytes_pinned, 0, "");
|
||||
|
||||
destroy_test_data(tsdn, &tdata);
|
||||
}
|
||||
TEST_END
|
||||
|
||||
TEST_BEGIN(test_max_bytes_too_low) {
|
||||
test_data_t tdata;
|
||||
sec_opts_t opts;
|
||||
|
|
@ -232,11 +330,11 @@ TEST_BEGIN(test_max_bytes_too_low) {
|
|||
edata_list_active_t allocs;
|
||||
edata_list_active_init(&allocs);
|
||||
edata_t edata1;
|
||||
edata_size_set(&edata1, 3 * PAGE);
|
||||
edata_init_test_size(&edata1, 3 * PAGE);
|
||||
edata_list_active_append(&allocs, &edata1);
|
||||
|
||||
/* SEC is empty, we return one pointer to it */
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_false(
|
||||
edata_list_active_empty(&allocs), "extents should not be consumed");
|
||||
destroy_test_data(tsdn, &tdata);
|
||||
|
|
@ -262,13 +360,13 @@ TEST_BEGIN(test_sec_flush) {
|
|||
edata_t edata1[NALLOCS];
|
||||
edata_t edata4[NALLOCS];
|
||||
for (int i = 0; i < NALLOCS; i++) {
|
||||
edata_size_set(&edata1[i], PAGE);
|
||||
edata_size_set(&edata4[i], 4 * PAGE);
|
||||
edata_init_test_size(&edata1[i], PAGE);
|
||||
edata_init_test_size(&edata4[i], 4 * PAGE);
|
||||
|
||||
edata_list_active_append(&allocs1, &edata1[i]);
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs1);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs1);
|
||||
edata_list_active_append(&allocs4, &edata4[i]);
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs4);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs4);
|
||||
}
|
||||
|
||||
sec_stats_t stats = {0};
|
||||
|
|
@ -301,23 +399,23 @@ TEST_BEGIN(test_sec_stats) {
|
|||
edata_list_active_t allocs;
|
||||
edata_list_active_init(&allocs);
|
||||
edata_t edata1;
|
||||
edata_size_set(&edata1, PAGE);
|
||||
edata_init_test_size(&edata1, PAGE);
|
||||
edata_list_active_append(&allocs, &edata1);
|
||||
|
||||
/* SEC is empty alloc fails. nmisses==1 */
|
||||
edata_t *edata = sec_alloc(tsdn, &tdata.sec, PAGE);
|
||||
edata_t *edata = sec_test_alloc(tsdn, &tdata.sec, PAGE);
|
||||
expect_ptr_null(edata, "SEC should be empty");
|
||||
|
||||
/* SEC is empty, we return one pointer to it. ndalloc_noflush=1 */
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_true(
|
||||
edata_list_active_empty(&allocs), "extents should be consumed");
|
||||
|
||||
edata_t edata2;
|
||||
edata_size_set(&edata2, PAGE);
|
||||
edata_init_test_size(&edata2, PAGE);
|
||||
edata_list_active_append(&allocs, &edata2);
|
||||
/* Sec can take one more, so ndalloc_noflush=2 */
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_true(
|
||||
edata_list_active_empty(&allocs), "extents should be consumed");
|
||||
|
||||
|
|
@ -335,9 +433,9 @@ TEST_BEGIN(test_sec_stats) {
|
|||
* the limit. This will force flush, so ndalloc_flush = 1.
|
||||
*/
|
||||
edata_t edata3;
|
||||
edata_size_set(&edata3, PAGE);
|
||||
edata_init_test_size(&edata3, PAGE);
|
||||
edata_list_active_append(&allocs, &edata3);
|
||||
sec_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
sec_test_dalloc(tsdn, &tdata.sec, &allocs);
|
||||
expect_false(
|
||||
edata_list_active_empty(&allocs), "extents should NOT be consumed");
|
||||
sec_stats_merge(tsdn, &tdata.sec, &stats);
|
||||
|
|
@ -351,11 +449,6 @@ TEST_END
|
|||
#define NOPS_PER_THREAD 100
|
||||
#define NPREFILL 32
|
||||
|
||||
static void
|
||||
edata_init_test(edata_t *edata) {
|
||||
memset(edata, 0, sizeof(*edata));
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
sec_t *sec;
|
||||
uint8_t preferred_shard;
|
||||
|
|
@ -379,12 +472,12 @@ thd_trylock_test(void *varg) {
|
|||
*shard_idx = arg->preferred_shard;
|
||||
|
||||
/* Fill the shard with some extents */
|
||||
sec_fill(tsdn, arg->sec, PAGE, &arg->fill_list, arg->fill_list_sz);
|
||||
sec_test_fill(tsdn, arg->sec, PAGE, &arg->fill_list, arg->fill_list_sz);
|
||||
expect_true(edata_list_active_empty(&arg->fill_list), "");
|
||||
|
||||
for (unsigned i = 0; i < NOPS_PER_THREAD; i++) {
|
||||
/* Try to allocate from SEC */
|
||||
arg->edata[i] = sec_alloc(tsdn, arg->sec, PAGE);
|
||||
arg->edata[i] = sec_test_alloc(tsdn, arg->sec, PAGE);
|
||||
if (arg->edata[i] != NULL) {
|
||||
expect_zu_eq(edata_size_get(arg->edata[i]), PAGE, "");
|
||||
}
|
||||
|
|
@ -397,7 +490,7 @@ thd_trylock_test(void *varg) {
|
|||
arg->nallocs++;
|
||||
edata_list_active_append(&list, arg->edata[i]);
|
||||
expect_zu_eq(edata_size_get(arg->edata[i]), PAGE, "");
|
||||
sec_dalloc(tsdn, arg->sec, &list);
|
||||
sec_test_dalloc(tsdn, arg->sec, &list);
|
||||
if (edata_list_active_empty(&list)) {
|
||||
arg->ndallocs++;
|
||||
} else {
|
||||
|
|
@ -433,8 +526,7 @@ TEST_BEGIN(test_sec_multishard) {
|
|||
edata_list_active_init(&args[i].fill_list);
|
||||
for (unsigned j = 0; j < NPREFILL; ++j) {
|
||||
size_t ind = i * NPREFILL + j;
|
||||
edata_init_test(&all_edatas[ind]);
|
||||
edata_size_set(&all_edatas[ind], PAGE);
|
||||
edata_init_test_size(&all_edatas[ind], PAGE);
|
||||
edata_list_active_append(
|
||||
&args[i].fill_list, &all_edatas[ind]);
|
||||
}
|
||||
|
|
@ -487,6 +579,6 @@ int
|
|||
main(void) {
|
||||
return test(test_max_nshards_option_zero,
|
||||
test_max_alloc_option_too_small, test_sec_fill, test_sec_alloc,
|
||||
test_sec_dalloc, test_max_bytes_too_low, test_sec_flush,
|
||||
test_sec_stats, test_sec_multishard);
|
||||
test_sec_dalloc, test_sec_pinned_stats, test_max_bytes_too_low,
|
||||
test_sec_flush, test_sec_stats, test_sec_multishard);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue