diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index e25c0b85..a2027966 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -25,6 +25,40 @@ typedef struct arena_s arena_t; typedef struct prof_tdata_s prof_tdata_t; +/* + * JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler + * cannot cache it across a user-space context switch. A raw `&tlsvar` is + * loop-invariant; inlined into malloc/free under LTO it can be hoisted across a + * swapcontext and reused on the OS thread a fiber migrated to -- a stale + * tsd/tcache. Route all tsd access through this macro; never take `&tsd_tls` + * raw. See https://github.com/jemalloc/jemalloc/issues/2890 + * + * The accessor takes the address behind a noinline `memory` barrier, opaque to + * the optimizer. MSVC has no inline asm, so it keeps the plain address; an MSVC + * build whose fibers run under /GL must instead compile with /GT (fiber-safe + * TLS). DECLARE is emitted in every TU; + * DEFINE (the out-of-line body) only under JEMALLOC_TSD_C_, i.e. once in + * src/tsd.c. Both must be invoked directly, not forwarded through another + * macro, or `##tlsvar` pastes the macro-expanded `je_tsd_tls` instead of the + * literal name. + */ +#if defined(__GNUC__) +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ + __typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void); +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \ + JEMALLOC_NOINLINE __typeof__(&(tlsvar)) \ + jemalloc_tls_addr_##tlsvar(void) { \ + __typeof__(&(tlsvar)) tls_addr = &(tlsvar); \ + __asm__ __volatile__("" : "+r"(tls_addr) : : "memory"); \ + return tls_addr; \ + } +# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar()) +#else +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) +# define JEMALLOC_TLS_ADDR(tlsvar) (&(tlsvar)) +#endif + /* * Thread-Specific-Data layout * diff --git a/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index 00756df1..cecdde94 100644 --- a/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -13,6 +13,13 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_tls) +JEMALLOC_TLS_ADDR_DECLARE(tsd_initialized) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_tls) +JEMALLOC_TLS_ADDR_DEFINE(tsd_initialized) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_cleanup_wrapper(void) { @@ -53,13 +60,15 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &tsd_tls; + return JEMALLOC_TLS_ADDR(tsd_tls); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls); + assert(tsd_booted); - if (likely(&tsd_tls != val)) { - tsd_tls = (*val); + if (likely(tsd != val)) { + *tsd = (*val); } - tsd_initialized = true; + *JEMALLOC_TLS_ADDR(tsd_initialized) = true; } diff --git a/include/jemalloc/internal/tsd_tls.h b/include/jemalloc/internal/tsd_tls.h index 6536eb54..043db3a8 100644 --- a/include/jemalloc/internal/tsd_tls.h +++ b/include/jemalloc/internal/tsd_tls.h @@ -13,6 +13,11 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern pthread_key_t tsd_tsd; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_tls) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_tls) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_boot0(void) { @@ -46,16 +51,18 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &tsd_tls; + return JEMALLOC_TLS_ADDR(tsd_tls); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls); + assert(tsd_booted); - if (likely(&tsd_tls != val)) { - tsd_tls = (*val); + if (likely(tsd != val)) { + *tsd = (*val); } - if (pthread_setspecific(tsd_tsd, (void *)(&tsd_tls)) != 0) { + if (pthread_setspecific(tsd_tsd, (void *)tsd) != 0) { malloc_write(": Error setting tsd.\n"); if (opt_abort) { abort(); diff --git a/include/jemalloc/internal/tsd_win.h b/include/jemalloc/internal/tsd_win.h index 8b22bec1..cd28a6e9 100644 --- a/include/jemalloc/internal/tsd_win.h +++ b/include/jemalloc/internal/tsd_win.h @@ -178,6 +178,11 @@ tsd_set(tsd_t *val) { extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_wrapper_tls) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_wrapper_tls) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_cleanup_wrapper(void) { @@ -223,16 +228,18 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &(tsd_wrapper_tls.val); + return &(JEMALLOC_TLS_ADDR(tsd_wrapper_tls)->val); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_wrapper_t *wrapper = JEMALLOC_TLS_ADDR(tsd_wrapper_tls); + assert(tsd_booted); - if (likely(&(tsd_wrapper_tls.val) != val)) { - tsd_wrapper_tls.val = (*val); + if (likely(&wrapper->val != val)) { + wrapper->val = (*val); } - tsd_wrapper_tls.initialized = true; + wrapper->initialized = true; } #endif // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) diff --git a/src/tsd.c b/src/tsd.c index 9c05e17f..b8f79044 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -1,3 +1,6 @@ +/* Emit the single out-of-line JEMALLOC_TLS_ADDR offset globals/helpers here. */ +#define JEMALLOC_TSD_C_ + #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/arenas_management.h"