mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-06-19 10:25:38 +03:00
Replace JET_WRAP_ with JET_EXTERN_INLINE macro
This commit is contained in:
Slobodan Predolac
parent
dfb276f035
commit
9f37c70826
3 changed files with 33 additions and 80 deletions
|
|
@ -39,9 +39,11 @@
|
|||
#ifdef JEMALLOC_JET
|
||||
# define JET_MUTABLE
|
||||
# define JET_EXTERN extern
|
||||
# define JET_EXTERN_INLINE extern
|
||||
#else
|
||||
# define JET_MUTABLE const
|
||||
# define JET_EXTERN static
|
||||
# define JET_EXTERN_INLINE static inline
|
||||
#endif
|
||||
|
||||
#define JEMALLOC_VA_ARGS_HEAD(head, ...) head
|
||||
|
|
|
|||
63
src/tcache.c
63
src/tcache.c
|
|
@ -148,7 +148,7 @@ tcache_bin_fill_ctl_get(tcache_slow_t *tcache_slow, szind_t szind) {
|
|||
* The base is adjusted during GC based on the traffic within a period of time,
|
||||
* while the offset is updated in real time to handle the immediate traffic.
|
||||
*/
|
||||
static inline uint8_t
|
||||
JET_EXTERN_INLINE uint8_t
|
||||
tcache_nfill_small_lg_div_get(tcache_slow_t *tcache_slow, szind_t szind) {
|
||||
cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
|
||||
return (ctl->base - (opt_experimental_tcache_gc ? ctl->offset : 0));
|
||||
|
|
@ -159,7 +159,7 @@ tcache_nfill_small_lg_div_get(tcache_slow_t *tcache_slow, szind_t szind) {
|
|||
* offset is increased so that (base - offset) is decreased,
|
||||
* which in return increases the number of items to be filled.
|
||||
*/
|
||||
static inline void
|
||||
JET_EXTERN_INLINE void
|
||||
tcache_nfill_small_burst_prepare(tcache_slow_t *tcache_slow, szind_t szind) {
|
||||
cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
|
||||
if (ctl->offset + 1 < ctl->base) {
|
||||
|
|
@ -167,7 +167,7 @@ tcache_nfill_small_burst_prepare(tcache_slow_t *tcache_slow, szind_t szind) {
|
|||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
JET_EXTERN_INLINE void
|
||||
tcache_nfill_small_burst_reset(tcache_slow_t *tcache_slow, szind_t szind) {
|
||||
cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
|
||||
ctl->offset = 0;
|
||||
|
|
@ -180,7 +180,7 @@ tcache_nfill_small_burst_reset(tcache_slow_t *tcache_slow, szind_t szind) {
|
|||
* limit != 0: limit is set to ncached_max, indicating that the fill
|
||||
* count should be decreased, i.e. lg_div(base) should be increased.
|
||||
*/
|
||||
static inline void
|
||||
JET_EXTERN_INLINE void
|
||||
tcache_nfill_small_gc_update(
|
||||
tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit) {
|
||||
cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind);
|
||||
|
|
@ -201,7 +201,7 @@ tcache_nfill_small_gc_update(
|
|||
ctl->offset = 0;
|
||||
}
|
||||
|
||||
static uint8_t
|
||||
JET_EXTERN uint8_t
|
||||
tcache_gc_item_delay_compute(szind_t szind) {
|
||||
assert(szind < SC_NBINS);
|
||||
size_t sz = sz_index2size(szind);
|
||||
|
|
@ -220,7 +220,7 @@ tcache_gc_is_addr_remote(void *addr, uintptr_t min, uintptr_t max) {
|
|||
return ((uintptr_t)addr < min || (uintptr_t)addr >= max);
|
||||
}
|
||||
|
||||
static inline cache_bin_sz_t
|
||||
JET_EXTERN_INLINE cache_bin_sz_t
|
||||
tcache_gc_small_nremote_get(cache_bin_t *cache_bin, void *addr,
|
||||
uintptr_t *addr_min, uintptr_t *addr_max, szind_t szind, size_t nflush) {
|
||||
assert(addr != NULL && addr_min != NULL && addr_max != NULL);
|
||||
|
|
@ -277,7 +277,7 @@ tcache_gc_small_nremote_get(cache_bin_t *cache_bin, void *addr,
|
|||
}
|
||||
|
||||
/* Shuffle the ptrs in the bin to put the remote pointers at the bottom. */
|
||||
static inline void
|
||||
JET_EXTERN_INLINE void
|
||||
tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
|
||||
uintptr_t addr_min, uintptr_t addr_max) {
|
||||
void **swap = NULL;
|
||||
|
|
@ -350,55 +350,6 @@ tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
|
|||
}
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_JET
|
||||
/*
|
||||
* The GC helpers above are static inline so they cannot be linked from a
|
||||
* separate translation unit. In JET builds we expose thin wrappers with a
|
||||
* `_test` suffix so test/unit/tcache_gc.c can exercise them directly. These
|
||||
* symbols are absent from the production library.
|
||||
*/
|
||||
#define JET_WRAP_RET(ret, fn, params, args) \
|
||||
ret fn##_test params { \
|
||||
return fn args; \
|
||||
}
|
||||
#define JET_WRAP_VOID(fn, params, args) \
|
||||
void fn##_test params { \
|
||||
fn args; \
|
||||
}
|
||||
|
||||
JET_WRAP_RET(cache_bin_sz_t, tcache_gc_small_nremote_get,
|
||||
(cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min,
|
||||
uintptr_t *addr_max, szind_t szind, size_t nflush),
|
||||
(cache_bin, addr, addr_min, addr_max, szind, nflush))
|
||||
|
||||
JET_WRAP_VOID(tcache_gc_small_bin_shuffle,
|
||||
(cache_bin_t *cache_bin, cache_bin_sz_t nremote,
|
||||
uintptr_t addr_min, uintptr_t addr_max),
|
||||
(cache_bin, nremote, addr_min, addr_max))
|
||||
|
||||
JET_WRAP_RET(uint8_t, tcache_nfill_small_lg_div_get,
|
||||
(tcache_slow_t *tcache_slow, szind_t szind),
|
||||
(tcache_slow, szind))
|
||||
|
||||
JET_WRAP_VOID(tcache_nfill_small_burst_prepare,
|
||||
(tcache_slow_t *tcache_slow, szind_t szind),
|
||||
(tcache_slow, szind))
|
||||
|
||||
JET_WRAP_VOID(tcache_nfill_small_burst_reset,
|
||||
(tcache_slow_t *tcache_slow, szind_t szind),
|
||||
(tcache_slow, szind))
|
||||
|
||||
JET_WRAP_VOID(tcache_nfill_small_gc_update,
|
||||
(tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit),
|
||||
(tcache_slow, szind, limit))
|
||||
|
||||
JET_WRAP_RET(uint8_t, tcache_gc_item_delay_compute,
|
||||
(szind_t szind), (szind))
|
||||
|
||||
#undef JET_WRAP_RET
|
||||
#undef JET_WRAP_VOID
|
||||
#endif
|
||||
|
||||
static bool
|
||||
tcache_gc_small(
|
||||
tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) {
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
#include "test/jemalloc_test.h"
|
||||
|
||||
extern cache_bin_sz_t tcache_gc_small_nremote_get_test(
|
||||
extern cache_bin_sz_t tcache_gc_small_nremote_get(
|
||||
cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min,
|
||||
uintptr_t *addr_max, szind_t szind, size_t nflush);
|
||||
extern void tcache_gc_small_bin_shuffle_test(cache_bin_t *cache_bin,
|
||||
extern void tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin,
|
||||
cache_bin_sz_t nremote, uintptr_t addr_min, uintptr_t addr_max);
|
||||
extern uint8_t tcache_nfill_small_lg_div_get_test(
|
||||
extern uint8_t tcache_nfill_small_lg_div_get(
|
||||
tcache_slow_t *tcache_slow, szind_t szind);
|
||||
extern void tcache_nfill_small_burst_prepare_test(
|
||||
extern void tcache_nfill_small_burst_prepare(
|
||||
tcache_slow_t *tcache_slow, szind_t szind);
|
||||
extern void tcache_nfill_small_burst_reset_test(
|
||||
extern void tcache_nfill_small_burst_reset(
|
||||
tcache_slow_t *tcache_slow, szind_t szind);
|
||||
extern void tcache_nfill_small_gc_update_test(
|
||||
extern void tcache_nfill_small_gc_update(
|
||||
tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit);
|
||||
extern uint8_t tcache_gc_item_delay_compute_test(szind_t szind);
|
||||
extern uint8_t tcache_gc_item_delay_compute(szind_t szind);
|
||||
|
||||
static void *
|
||||
test_cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info,
|
||||
|
|
@ -65,7 +65,7 @@ TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) {
|
|||
|
||||
uintptr_t addr_min;
|
||||
uintptr_t addr_max;
|
||||
cache_bin_sz_t nremote = tcache_gc_small_nremote_get_test(&bin,
|
||||
cache_bin_sz_t nremote = tcache_gc_small_nremote_get(&bin,
|
||||
(void *)anchor, &addr_min, &addr_max, szind, 2);
|
||||
expect_zu_eq(2, nremote,
|
||||
"Should count pointers outside the local slab");
|
||||
|
|
@ -73,7 +73,7 @@ TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) {
|
|||
expect_zu_eq(anchor + slab_size, addr_max,
|
||||
"Expected slab-local upper bound");
|
||||
|
||||
tcache_gc_small_bin_shuffle_test(&bin, nremote, addr_min, addr_max);
|
||||
tcache_gc_small_bin_shuffle(&bin, nremote, addr_min, addr_max);
|
||||
expect_ptr_eq(ptrs[0], bin.stack_head[0],
|
||||
"Local pointer order should be preserved");
|
||||
expect_ptr_eq(ptrs[2], bin.stack_head[1],
|
||||
|
|
@ -89,7 +89,7 @@ TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) {
|
|||
cache_bin_alloc(&bin, &success);
|
||||
}
|
||||
cache_bin_fill_ptrs(&bin, ptrs, 4);
|
||||
nremote = tcache_gc_small_nremote_get_test(&bin, (void *)anchor,
|
||||
nremote = tcache_gc_small_nremote_get(&bin, (void *)anchor,
|
||||
&addr_min, &addr_max, szind, 1);
|
||||
expect_zu_eq(1, nremote,
|
||||
"Neighbor filtering should be used when it satisfies nflush");
|
||||
|
|
@ -116,42 +116,42 @@ TEST_BEGIN(test_tcache_gc_fill_control_and_delay) {
|
|||
size_t old_tcache_gc_delay_bytes = opt_tcache_gc_delay_bytes;
|
||||
|
||||
opt_experimental_tcache_gc = true;
|
||||
expect_u_eq(3, tcache_nfill_small_lg_div_get_test(
|
||||
expect_u_eq(3, tcache_nfill_small_lg_div_get(
|
||||
&tcache_slow, szind), "Unexpected initial fill divisor");
|
||||
tcache_nfill_small_burst_prepare_test(&tcache_slow, szind);
|
||||
expect_u_eq(2, tcache_nfill_small_lg_div_get_test(
|
||||
tcache_nfill_small_burst_prepare(&tcache_slow, szind);
|
||||
expect_u_eq(2, tcache_nfill_small_lg_div_get(
|
||||
&tcache_slow, szind), "Burst load should increase fill count");
|
||||
tcache_nfill_small_burst_prepare_test(&tcache_slow, szind);
|
||||
expect_u_eq(1, tcache_nfill_small_lg_div_get_test(
|
||||
tcache_nfill_small_burst_prepare(&tcache_slow, szind);
|
||||
expect_u_eq(1, tcache_nfill_small_lg_div_get(
|
||||
&tcache_slow, szind), "Burst load should cap at divisor 1");
|
||||
tcache_nfill_small_burst_prepare_test(&tcache_slow, szind);
|
||||
expect_u_eq(1, tcache_nfill_small_lg_div_get_test(
|
||||
tcache_nfill_small_burst_prepare(&tcache_slow, szind);
|
||||
expect_u_eq(1, tcache_nfill_small_lg_div_get(
|
||||
&tcache_slow, szind), "Burst offset should not reach base");
|
||||
|
||||
tcache_nfill_small_burst_reset_test(&tcache_slow, szind);
|
||||
expect_u_eq(3, tcache_nfill_small_lg_div_get_test(
|
||||
tcache_nfill_small_burst_reset(&tcache_slow, szind);
|
||||
expect_u_eq(3, tcache_nfill_small_lg_div_get(
|
||||
&tcache_slow, szind), "Burst reset should clear offset");
|
||||
|
||||
tcache_nfill_small_gc_update_test(&tcache_slow, szind, 0);
|
||||
tcache_nfill_small_gc_update(&tcache_slow, szind, 0);
|
||||
expect_u_eq(2, ctl->base,
|
||||
"Refill during a GC period should increase future fill count");
|
||||
expect_u_eq(0, ctl->offset, "GC update should reset burst offset");
|
||||
|
||||
tcache_nfill_small_gc_update_test(&tcache_slow, szind, 64);
|
||||
tcache_nfill_small_gc_update(&tcache_slow, szind, 64);
|
||||
expect_u_eq(3, ctl->base,
|
||||
"Low-water pressure should reduce future fill count");
|
||||
|
||||
ctl->offset = 2;
|
||||
opt_experimental_tcache_gc = false;
|
||||
expect_u_eq(3, tcache_nfill_small_lg_div_get_test(
|
||||
expect_u_eq(3, tcache_nfill_small_lg_div_get(
|
||||
&tcache_slow, szind), "Legacy GC should ignore burst offset");
|
||||
|
||||
size_t sz = sz_index2size(szind);
|
||||
opt_tcache_gc_delay_bytes = 3 * sz;
|
||||
expect_u_eq(3, tcache_gc_item_delay_compute_test(szind),
|
||||
expect_u_eq(3, tcache_gc_item_delay_compute(szind),
|
||||
"Delay should convert bytes to items");
|
||||
opt_tcache_gc_delay_bytes = SIZE_T_MAX;
|
||||
expect_u_eq(UINT8_MAX, tcache_gc_item_delay_compute_test(szind),
|
||||
expect_u_eq(UINT8_MAX, tcache_gc_item_delay_compute(szind),
|
||||
"Delay should saturate at uint8 max");
|
||||
|
||||
opt_experimental_tcache_gc = old_experimental_tcache_gc;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue