From 7aa77c8e2a897ae5f56ca36ad9b32f0f2a00e72b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 22 Jun 2026 12:47:46 +0200 Subject: [PATCH] Add an inline fast path to `JEMALLOC_TLS_ADDR` for static TLS The noinline accessor puts an opaque call on the malloc/free fastpath. test/stress/microbench (malloc(1)/free pairs, pinned core, clang): unpatched accessor this commit malloc, no LTO 7.0 ns/op 9.8 (+40%) 7.0 (+-0%) free, no LTO 6.5 ns/op 9.2 (+40%) 6.5 (+-0%) malloc, ThinLTO 7.0 ns/op 9.0 (+28%) 7.6 (~+8%) free, ThinLTO 6.5 ns/op 8.4 (+28%) 7.0 (~+8%) Under a static TLS model the tsd address is `thread_pointer + offset` with a thread-independent offset. Capture the offset once (a noinline helper, lazy and sentinel-initialized) and re-read the thread pointer per call with a `volatile` asm the optimizer may not hoist or CSE (per-arch reads from mimalloc's mi_prim_tls_slot). Both inputs are hoist-proof: the tp read is volatile, the offset is thread-independent. The offset global is read and written with relaxed atomics: threads racing their first allocation init it concurrently with the same value, so a plain access would be a benign-but-UB data race (TSan-reported). Relaxed adds no fence -- it compiles to the same load/store on the fast path. The offset must NOT be computed inline as `&tsd_tls - __builtin_thread_pointer()`: the terms hoist independently, and clang ThinLTO keeps the stale `&tsd_tls` in a callee-saved register while re-reading the thread pointer, so `fresh_tp + (stale_addr - fresh_tp)` cancels back to the stale address. Capturing behind a call boundary evaluates both terms at one point on one thread. Gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC (a new configure define, set whenever jemalloc applies its default initial-exec model), GNU asm, a known arch, and !_WIN32 (MinGW's thread pointer lives in the TEB, not fs/gs:0); everything else keeps the noinline accessor. Verified with the fiber-migration reproducer under clang ThinLTO. Co-Authored-By: Claude Opus 4.8 (1M context) --- configure.ac | 2 + .../internal/jemalloc_internal_defs.h.in | 7 ++ include/jemalloc/internal/tsd_internals.h | 83 ++++++++++++++++--- 3 files changed, 79 insertions(+), 13 deletions(-) diff --git a/configure.ac b/configure.ac index 12af2e99..9dcfd8f4 100644 --- a/configure.ac +++ b/configure.ac @@ -2841,6 +2841,8 @@ if test "x${je_cv_tls_model}" = "xyes" -a \ AC_DEFINE([JEMALLOC_TLS_MODEL], [__attribute__((tls_model("initial-exec")))], [ ]) + AC_DEFINE([JEMALLOC_TLS_MODEL_INITIAL_EXEC], [ ], + [Defined when the TSD thread-locals use the initial-exec (static) TLS model, i.e. live at a fixed offset from the thread pointer.]) else AC_DEFINE([JEMALLOC_TLS_MODEL], [ ], [ ]) fi diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 0b750a99..231a0d75 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -148,6 +148,13 @@ /* Non-empty if the tls_model attribute is supported. */ #undef JEMALLOC_TLS_MODEL +/* + * Defined when the TSD thread-locals use the initial-exec (static) TLS model, + * i.e. live at a fixed offset from the thread pointer. Gates the fast path of + * JEMALLOC_TLS_ADDR (see tsd_internals.h). + */ +#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC + /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables * inline functions. diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index a2027966..34af60f4 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -28,21 +28,78 @@ typedef struct prof_tdata_s prof_tdata_t; /* * JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler * cannot cache it across a user-space context switch. A raw `&tlsvar` is - * loop-invariant; inlined into malloc/free under LTO it can be hoisted across a - * swapcontext and reused on the OS thread a fiber migrated to -- a stale - * tsd/tcache. Route all tsd access through this macro; never take `&tsd_tls` - * raw. See https://github.com/jemalloc/jemalloc/issues/2890 + * `thread_pointer + const_offset`, loop-invariant; inlined into malloc/free + * under LTO it can be hoisted across a swapcontext and reused on the OS thread a + * fiber migrated to -- a stale tsd/tcache. Route all tsd access through this + * macro; never take `&tsd_tls` raw. See + * https://github.com/jemalloc/jemalloc/issues/2890 * - * The accessor takes the address behind a noinline `memory` barrier, opaque to - * the optimizer. MSVC has no inline asm, so it keeps the plain address; an MSVC - * build whose fibers run under /GL must instead compile with /GT (fiber-safe - * TLS). DECLARE is emitted in every TU; - * DEFINE (the out-of-line body) only under JEMALLOC_TSD_C_, i.e. once in - * src/tsd.c. Both must be invoked directly, not forwarded through another - * macro, or `##tlsvar` pastes the macro-expanded `je_tsd_tls` instead of the - * literal name. + * Static-TLS fast path (gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC): re-read the + * thread pointer with a volatile asm and add a runtime-captured constant offset. + * The offset MUST be captured by the noinline helper, not computed inline as + * `&tlsvar - thread_pointer` -- inline, the two terms hoist independently and + * cancel the volatile read back to the stale address. The offset is read and + * written with relaxed atomics -- threads racing their first allocation init it + * concurrently with the same thread-independent value. Other configs use a + * noinline `memory`-barrier accessor. MSVC has no inline asm, so it keeps the + * plain address; an MSVC build whose fibers run under whole-program opt (/GL) + * must instead compile with /GT (fiber-safe TLS). + * + * DECLARE is emitted in every TU; DEFINE (the out-of-line bodies) only under + * JEMALLOC_TSD_C_, i.e. once in src/tsd.c. Both must be invoked directly, not + * forwarded through another macro, or `##tlsvar` pastes the macro-expanded + * `je_tsd_tls` instead of the literal name. */ -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(_WIN32) && \ + defined(JEMALLOC_TLS_MODEL_INITIAL_EXEC) && \ + (defined(__aarch64__) || defined(__arm__) || defined(__x86_64__) || \ + defined(__i386__)) +JEMALLOC_ALWAYS_INLINE char * +jemalloc_thread_pointer(void) { + char *thread_pointer; +# if defined(__aarch64__) && defined(__APPLE__) + __asm__ __volatile__("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r"(thread_pointer)); +# elif defined(__aarch64__) + __asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(thread_pointer)); +# elif defined(__arm__) + __asm__ __volatile__("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r"(thread_pointer)); +# elif defined(__x86_64__) && defined(__APPLE__) + __asm__ __volatile__("movq %%gs:0, %0" : "=r"(thread_pointer)); +# elif defined(__x86_64__) + __asm__ __volatile__("movq %%fs:0, %0" : "=r"(thread_pointer)); +# else /* __i386__ */ + __asm__ __volatile__("movl %%gs:0, %0" : "=r"(thread_pointer)); +# endif + return thread_pointer; +} +/* 1 is unreachable: tlsvar and the thread pointer are at least 4-aligned. */ +# define JEMALLOC_TLS_OFFSET_UNINITIALIZED 1 +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ + extern intptr_t jemalloc_tls_offset_##tlsvar; \ + intptr_t jemalloc_tls_offset_init_##tlsvar(void); \ + JEMALLOC_ALWAYS_INLINE __typeof__(&(tlsvar)) \ + jemalloc_tls_addr_##tlsvar(void) { \ + intptr_t tls_offset = __atomic_load_n(&jemalloc_tls_offset_##tlsvar, \ + __ATOMIC_RELAXED); \ + if (unlikely(tls_offset == JEMALLOC_TLS_OFFSET_UNINITIALIZED)) { \ + tls_offset = jemalloc_tls_offset_init_##tlsvar(); \ + } \ + return (__typeof__(&(tlsvar)))(jemalloc_thread_pointer() + \ + tls_offset); \ + } +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \ + intptr_t jemalloc_tls_offset_##tlsvar = \ + JEMALLOC_TLS_OFFSET_UNINITIALIZED; \ + JEMALLOC_NOINLINE intptr_t \ + jemalloc_tls_offset_init_##tlsvar(void) { \ + intptr_t tls_offset = (intptr_t)((char *)&(tlsvar) - \ + jemalloc_thread_pointer()); \ + __atomic_store_n(&jemalloc_tls_offset_##tlsvar, tls_offset, \ + __ATOMIC_RELAXED); \ + return tls_offset; \ + } +# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar()) +#elif defined(__GNUC__) # define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ __typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void); # define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \