From 7b9853c4a1dbf9fc21f2acc94040691ce5f5af66 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 22 Jun 2026 12:47:46 +0200 Subject: [PATCH] Guard TSD thread-locals against LTO hoisting (`JEMALLOC_TLS_ADDR`) `tsd_get`/`tsd_set` took the tsd address as a plain `&tsd_tls` -- `thread_pointer + const_offset`, which the compiler treats as loop-invariant. Under whole-program LTO it is hoisted out of the inlined malloc/free and kept in a callee-saved register across a user-space context switch (swapcontext, boost::context fibers); resumed on a different OS thread, the cached tsd/tcache still belongs to the previous thread and the two race -- heap corruption that reproduces only under LTO and is invisible to sanitizers. The write side is worse: tsd_set (from tsd_fetch_slow, on a thread's first allocation after migration) would copy the new thread's tsd over the previous thread's live one and register the wrong cleanup key. Route tsd_get/tsd_set in every GNU backend through JEMALLOC_TLS_ADDR(tsd_tls): a per-variable accessor that takes the address inside a noinline `memory`-barrier function, opaque to the optimizer. Correct for every TLS model, one call per access; MSVC keeps the plain address. The accessor is declared in every TU (JEMALLOC_TLS_ADDR_DECLARE) but defined once in src/tsd.c (JEMALLOC_TLS_ADDR_DEFINE, under JEMALLOC_TSD_C_): a per-TU `static` body is emitted at -O0 even when unused, pulling an undefined reference to the internal thread-local into the integration-test util objects, which link only the public shared library. DECLARE/DEFINE are invoked directly, never forwarded, so `##tlsvar` pastes the literal name rather than the expanded `je_tsd_tls`. --- include/jemalloc/internal/tsd_internals.h | 34 +++++++++++++++++++ .../internal/tsd_malloc_thread_cleanup.h | 17 +++++++--- include/jemalloc/internal/tsd_tls.h | 15 +++++--- include/jemalloc/internal/tsd_win.h | 15 +++++--- src/tsd.c | 3 ++ 5 files changed, 72 insertions(+), 12 deletions(-) diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index e25c0b85..a2027966 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -25,6 +25,40 @@ typedef struct arena_s arena_t; typedef struct prof_tdata_s prof_tdata_t; +/* + * JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler + * cannot cache it across a user-space context switch. A raw `&tlsvar` is + * loop-invariant; inlined into malloc/free under LTO it can be hoisted across a + * swapcontext and reused on the OS thread a fiber migrated to -- a stale + * tsd/tcache. Route all tsd access through this macro; never take `&tsd_tls` + * raw. See https://github.com/jemalloc/jemalloc/issues/2890 + * + * The accessor takes the address behind a noinline `memory` barrier, opaque to + * the optimizer. MSVC has no inline asm, so it keeps the plain address; an MSVC + * build whose fibers run under /GL must instead compile with /GT (fiber-safe + * TLS). DECLARE is emitted in every TU; + * DEFINE (the out-of-line body) only under JEMALLOC_TSD_C_, i.e. once in + * src/tsd.c. Both must be invoked directly, not forwarded through another + * macro, or `##tlsvar` pastes the macro-expanded `je_tsd_tls` instead of the + * literal name. + */ +#if defined(__GNUC__) +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ + __typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void); +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \ + JEMALLOC_NOINLINE __typeof__(&(tlsvar)) \ + jemalloc_tls_addr_##tlsvar(void) { \ + __typeof__(&(tlsvar)) tls_addr = &(tlsvar); \ + __asm__ __volatile__("" : "+r"(tls_addr) : : "memory"); \ + return tls_addr; \ + } +# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar()) +#else +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) +# define JEMALLOC_TLS_ADDR(tlsvar) (&(tlsvar)) +#endif + /* * Thread-Specific-Data layout * diff --git a/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index 00756df1..cecdde94 100644 --- a/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -13,6 +13,13 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_tls) +JEMALLOC_TLS_ADDR_DECLARE(tsd_initialized) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_tls) +JEMALLOC_TLS_ADDR_DEFINE(tsd_initialized) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_cleanup_wrapper(void) { @@ -53,13 +60,15 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &tsd_tls; + return JEMALLOC_TLS_ADDR(tsd_tls); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls); + assert(tsd_booted); - if (likely(&tsd_tls != val)) { - tsd_tls = (*val); + if (likely(tsd != val)) { + *tsd = (*val); } - tsd_initialized = true; + *JEMALLOC_TLS_ADDR(tsd_initialized) = true; } diff --git a/include/jemalloc/internal/tsd_tls.h b/include/jemalloc/internal/tsd_tls.h index 6536eb54..043db3a8 100644 --- a/include/jemalloc/internal/tsd_tls.h +++ b/include/jemalloc/internal/tsd_tls.h @@ -13,6 +13,11 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern pthread_key_t tsd_tsd; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_tls) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_tls) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_boot0(void) { @@ -46,16 +51,18 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &tsd_tls; + return JEMALLOC_TLS_ADDR(tsd_tls); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls); + assert(tsd_booted); - if (likely(&tsd_tls != val)) { - tsd_tls = (*val); + if (likely(tsd != val)) { + *tsd = (*val); } - if (pthread_setspecific(tsd_tsd, (void *)(&tsd_tls)) != 0) { + if (pthread_setspecific(tsd_tsd, (void *)tsd) != 0) { malloc_write(": Error setting tsd.\n"); if (opt_abort) { abort(); diff --git a/include/jemalloc/internal/tsd_win.h b/include/jemalloc/internal/tsd_win.h index 8b22bec1..cd28a6e9 100644 --- a/include/jemalloc/internal/tsd_win.h +++ b/include/jemalloc/internal/tsd_win.h @@ -178,6 +178,11 @@ tsd_set(tsd_t *val) { extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_wrapper_tls) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_wrapper_tls) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_cleanup_wrapper(void) { @@ -223,16 +228,18 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &(tsd_wrapper_tls.val); + return &(JEMALLOC_TLS_ADDR(tsd_wrapper_tls)->val); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_wrapper_t *wrapper = JEMALLOC_TLS_ADDR(tsd_wrapper_tls); + assert(tsd_booted); - if (likely(&(tsd_wrapper_tls.val) != val)) { - tsd_wrapper_tls.val = (*val); + if (likely(&wrapper->val != val)) { + wrapper->val = (*val); } - tsd_wrapper_tls.initialized = true; + wrapper->initialized = true; } #endif // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) diff --git a/src/tsd.c b/src/tsd.c index 9c05e17f..b8f79044 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -1,3 +1,6 @@ +/* Emit the single out-of-line JEMALLOC_TLS_ADDR offset globals/helpers here. */ +#define JEMALLOC_TSD_C_ + #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/arenas_management.h"