From 9f37c70826a21022451aed1a1827d60c00c24226 Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Wed, 17 Jun 2026 10:51:49 -0700 Subject: [PATCH] Replace JET_WRAP_ with JET_EXTERN_INLINE macro --- .../internal/jemalloc_internal_macros.h | 2 + src/tcache.c | 63 +++---------------- test/unit/tcache_gc.c | 48 +++++++------- 3 files changed, 33 insertions(+), 80 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h index eb1ca119..f77423d6 100644 --- a/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/include/jemalloc/internal/jemalloc_internal_macros.h @@ -39,9 +39,11 @@ #ifdef JEMALLOC_JET # define JET_MUTABLE # define JET_EXTERN extern +# define JET_EXTERN_INLINE extern #else # define JET_MUTABLE const # define JET_EXTERN static +# define JET_EXTERN_INLINE static inline #endif #define JEMALLOC_VA_ARGS_HEAD(head, ...) head diff --git a/src/tcache.c b/src/tcache.c index 14d82a74..5f6b8239 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -148,7 +148,7 @@ tcache_bin_fill_ctl_get(tcache_slow_t *tcache_slow, szind_t szind) { * The base is adjusted during GC based on the traffic within a period of time, * while the offset is updated in real time to handle the immediate traffic. */ -static inline uint8_t +JET_EXTERN_INLINE uint8_t tcache_nfill_small_lg_div_get(tcache_slow_t *tcache_slow, szind_t szind) { cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); return (ctl->base - (opt_experimental_tcache_gc ? ctl->offset : 0)); @@ -159,7 +159,7 @@ tcache_nfill_small_lg_div_get(tcache_slow_t *tcache_slow, szind_t szind) { * offset is increased so that (base - offset) is decreased, * which in return increases the number of items to be filled. */ -static inline void +JET_EXTERN_INLINE void tcache_nfill_small_burst_prepare(tcache_slow_t *tcache_slow, szind_t szind) { cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); if (ctl->offset + 1 < ctl->base) { @@ -167,7 +167,7 @@ tcache_nfill_small_burst_prepare(tcache_slow_t *tcache_slow, szind_t szind) { } } -static inline void +JET_EXTERN_INLINE void tcache_nfill_small_burst_reset(tcache_slow_t *tcache_slow, szind_t szind) { cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); ctl->offset = 0; @@ -180,7 +180,7 @@ tcache_nfill_small_burst_reset(tcache_slow_t *tcache_slow, szind_t szind) { * limit != 0: limit is set to ncached_max, indicating that the fill * count should be decreased, i.e. lg_div(base) should be increased. */ -static inline void +JET_EXTERN_INLINE void tcache_nfill_small_gc_update( tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit) { cache_bin_fill_ctl_t *ctl = tcache_bin_fill_ctl_get(tcache_slow, szind); @@ -201,7 +201,7 @@ tcache_nfill_small_gc_update( ctl->offset = 0; } -static uint8_t +JET_EXTERN uint8_t tcache_gc_item_delay_compute(szind_t szind) { assert(szind < SC_NBINS); size_t sz = sz_index2size(szind); @@ -220,7 +220,7 @@ tcache_gc_is_addr_remote(void *addr, uintptr_t min, uintptr_t max) { return ((uintptr_t)addr < min || (uintptr_t)addr >= max); } -static inline cache_bin_sz_t +JET_EXTERN_INLINE cache_bin_sz_t tcache_gc_small_nremote_get(cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min, uintptr_t *addr_max, szind_t szind, size_t nflush) { assert(addr != NULL && addr_min != NULL && addr_max != NULL); @@ -277,7 +277,7 @@ tcache_gc_small_nremote_get(cache_bin_t *cache_bin, void *addr, } /* Shuffle the ptrs in the bin to put the remote pointers at the bottom. */ -static inline void +JET_EXTERN_INLINE void tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote, uintptr_t addr_min, uintptr_t addr_max) { void **swap = NULL; @@ -350,55 +350,6 @@ tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote, } } -#ifdef JEMALLOC_JET -/* - * The GC helpers above are static inline so they cannot be linked from a - * separate translation unit. In JET builds we expose thin wrappers with a - * `_test` suffix so test/unit/tcache_gc.c can exercise them directly. These - * symbols are absent from the production library. - */ -#define JET_WRAP_RET(ret, fn, params, args) \ - ret fn##_test params { \ - return fn args; \ - } -#define JET_WRAP_VOID(fn, params, args) \ - void fn##_test params { \ - fn args; \ - } - -JET_WRAP_RET(cache_bin_sz_t, tcache_gc_small_nremote_get, - (cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min, - uintptr_t *addr_max, szind_t szind, size_t nflush), - (cache_bin, addr, addr_min, addr_max, szind, nflush)) - -JET_WRAP_VOID(tcache_gc_small_bin_shuffle, - (cache_bin_t *cache_bin, cache_bin_sz_t nremote, - uintptr_t addr_min, uintptr_t addr_max), - (cache_bin, nremote, addr_min, addr_max)) - -JET_WRAP_RET(uint8_t, tcache_nfill_small_lg_div_get, - (tcache_slow_t *tcache_slow, szind_t szind), - (tcache_slow, szind)) - -JET_WRAP_VOID(tcache_nfill_small_burst_prepare, - (tcache_slow_t *tcache_slow, szind_t szind), - (tcache_slow, szind)) - -JET_WRAP_VOID(tcache_nfill_small_burst_reset, - (tcache_slow_t *tcache_slow, szind_t szind), - (tcache_slow, szind)) - -JET_WRAP_VOID(tcache_nfill_small_gc_update, - (tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit), - (tcache_slow, szind, limit)) - -JET_WRAP_RET(uint8_t, tcache_gc_item_delay_compute, - (szind_t szind), (szind)) - -#undef JET_WRAP_RET -#undef JET_WRAP_VOID -#endif - static bool tcache_gc_small( tsd_t *tsd, tcache_slow_t *tcache_slow, tcache_t *tcache, szind_t szind) { diff --git a/test/unit/tcache_gc.c b/test/unit/tcache_gc.c index 7257e70a..5d13638c 100644 --- a/test/unit/tcache_gc.c +++ b/test/unit/tcache_gc.c @@ -1,19 +1,19 @@ #include "test/jemalloc_test.h" -extern cache_bin_sz_t tcache_gc_small_nremote_get_test( +extern cache_bin_sz_t tcache_gc_small_nremote_get( cache_bin_t *cache_bin, void *addr, uintptr_t *addr_min, uintptr_t *addr_max, szind_t szind, size_t nflush); -extern void tcache_gc_small_bin_shuffle_test(cache_bin_t *cache_bin, +extern void tcache_gc_small_bin_shuffle(cache_bin_t *cache_bin, cache_bin_sz_t nremote, uintptr_t addr_min, uintptr_t addr_max); -extern uint8_t tcache_nfill_small_lg_div_get_test( +extern uint8_t tcache_nfill_small_lg_div_get( tcache_slow_t *tcache_slow, szind_t szind); -extern void tcache_nfill_small_burst_prepare_test( +extern void tcache_nfill_small_burst_prepare( tcache_slow_t *tcache_slow, szind_t szind); -extern void tcache_nfill_small_burst_reset_test( +extern void tcache_nfill_small_burst_reset( tcache_slow_t *tcache_slow, szind_t szind); -extern void tcache_nfill_small_gc_update_test( +extern void tcache_nfill_small_gc_update( tcache_slow_t *tcache_slow, szind_t szind, cache_bin_sz_t limit); -extern uint8_t tcache_gc_item_delay_compute_test(szind_t szind); +extern uint8_t tcache_gc_item_delay_compute(szind_t szind); static void * test_cache_bin_init(cache_bin_t *bin, cache_bin_info_t *info, @@ -65,7 +65,7 @@ TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) { uintptr_t addr_min; uintptr_t addr_max; - cache_bin_sz_t nremote = tcache_gc_small_nremote_get_test(&bin, + cache_bin_sz_t nremote = tcache_gc_small_nremote_get(&bin, (void *)anchor, &addr_min, &addr_max, szind, 2); expect_zu_eq(2, nremote, "Should count pointers outside the local slab"); @@ -73,7 +73,7 @@ TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) { expect_zu_eq(anchor + slab_size, addr_max, "Expected slab-local upper bound"); - tcache_gc_small_bin_shuffle_test(&bin, nremote, addr_min, addr_max); + tcache_gc_small_bin_shuffle(&bin, nremote, addr_min, addr_max); expect_ptr_eq(ptrs[0], bin.stack_head[0], "Local pointer order should be preserved"); expect_ptr_eq(ptrs[2], bin.stack_head[1], @@ -89,7 +89,7 @@ TEST_BEGIN(test_tcache_gc_small_remote_count_and_shuffle) { cache_bin_alloc(&bin, &success); } cache_bin_fill_ptrs(&bin, ptrs, 4); - nremote = tcache_gc_small_nremote_get_test(&bin, (void *)anchor, + nremote = tcache_gc_small_nremote_get(&bin, (void *)anchor, &addr_min, &addr_max, szind, 1); expect_zu_eq(1, nremote, "Neighbor filtering should be used when it satisfies nflush"); @@ -116,42 +116,42 @@ TEST_BEGIN(test_tcache_gc_fill_control_and_delay) { size_t old_tcache_gc_delay_bytes = opt_tcache_gc_delay_bytes; opt_experimental_tcache_gc = true; - expect_u_eq(3, tcache_nfill_small_lg_div_get_test( + expect_u_eq(3, tcache_nfill_small_lg_div_get( &tcache_slow, szind), "Unexpected initial fill divisor"); - tcache_nfill_small_burst_prepare_test(&tcache_slow, szind); - expect_u_eq(2, tcache_nfill_small_lg_div_get_test( + tcache_nfill_small_burst_prepare(&tcache_slow, szind); + expect_u_eq(2, tcache_nfill_small_lg_div_get( &tcache_slow, szind), "Burst load should increase fill count"); - tcache_nfill_small_burst_prepare_test(&tcache_slow, szind); - expect_u_eq(1, tcache_nfill_small_lg_div_get_test( + tcache_nfill_small_burst_prepare(&tcache_slow, szind); + expect_u_eq(1, tcache_nfill_small_lg_div_get( &tcache_slow, szind), "Burst load should cap at divisor 1"); - tcache_nfill_small_burst_prepare_test(&tcache_slow, szind); - expect_u_eq(1, tcache_nfill_small_lg_div_get_test( + tcache_nfill_small_burst_prepare(&tcache_slow, szind); + expect_u_eq(1, tcache_nfill_small_lg_div_get( &tcache_slow, szind), "Burst offset should not reach base"); - tcache_nfill_small_burst_reset_test(&tcache_slow, szind); - expect_u_eq(3, tcache_nfill_small_lg_div_get_test( + tcache_nfill_small_burst_reset(&tcache_slow, szind); + expect_u_eq(3, tcache_nfill_small_lg_div_get( &tcache_slow, szind), "Burst reset should clear offset"); - tcache_nfill_small_gc_update_test(&tcache_slow, szind, 0); + tcache_nfill_small_gc_update(&tcache_slow, szind, 0); expect_u_eq(2, ctl->base, "Refill during a GC period should increase future fill count"); expect_u_eq(0, ctl->offset, "GC update should reset burst offset"); - tcache_nfill_small_gc_update_test(&tcache_slow, szind, 64); + tcache_nfill_small_gc_update(&tcache_slow, szind, 64); expect_u_eq(3, ctl->base, "Low-water pressure should reduce future fill count"); ctl->offset = 2; opt_experimental_tcache_gc = false; - expect_u_eq(3, tcache_nfill_small_lg_div_get_test( + expect_u_eq(3, tcache_nfill_small_lg_div_get( &tcache_slow, szind), "Legacy GC should ignore burst offset"); size_t sz = sz_index2size(szind); opt_tcache_gc_delay_bytes = 3 * sz; - expect_u_eq(3, tcache_gc_item_delay_compute_test(szind), + expect_u_eq(3, tcache_gc_item_delay_compute(szind), "Delay should convert bytes to items"); opt_tcache_gc_delay_bytes = SIZE_T_MAX; - expect_u_eq(UINT8_MAX, tcache_gc_item_delay_compute_test(szind), + expect_u_eq(UINT8_MAX, tcache_gc_item_delay_compute(szind), "Delay should saturate at uint8 max"); opt_experimental_tcache_gc = old_experimental_tcache_gc;