Guard TSD thread-locals against LTO hoisting (JEMALLOC_TLS_ADDR)

`tsd_get`/`tsd_set` took the tsd address as a plain `&tsd_tls` -- `thread_pointer
+ const_offset`, which the compiler treats as loop-invariant.  Under
whole-program LTO it is hoisted out of the inlined malloc/free and kept in a
callee-saved register across a user-space context switch (swapcontext,
boost::context fibers); resumed on a different OS thread, the cached tsd/tcache
still belongs to the previous thread and the two race -- heap corruption that
reproduces only under LTO and is invisible to sanitizers.  The write side is
worse: tsd_set (from tsd_fetch_slow, on a thread's first allocation after
migration) would copy the new thread's tsd over the previous thread's live one
and register the wrong cleanup key.

Route tsd_get/tsd_set in every GNU backend through JEMALLOC_TLS_ADDR(tsd_tls): a
per-variable accessor that takes the address inside a noinline `memory`-barrier
function, opaque to the optimizer.  Correct for every TLS model, one call per
access; MSVC keeps the plain address.

The accessor is declared in every TU (JEMALLOC_TLS_ADDR_DECLARE) but defined
once in src/tsd.c (JEMALLOC_TLS_ADDR_DEFINE, under JEMALLOC_TSD_C_): a per-TU
`static` body is emitted at -O0 even when unused, pulling an undefined reference
to the internal thread-local into the integration-test util objects, which link
only the public shared library.  DECLARE/DEFINE are invoked directly, never
forwarded, so `##tlsvar` pastes the literal name rather than the expanded
`je_tsd_tls`.

include/jemalloc/internal/tsd_internals.h

@@ -25,6 +25,40 @@
 typedef struct arena_s      arena_t;
 typedef struct prof_tdata_s prof_tdata_t;
 
+/*
+ * JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler
+ * cannot cache it across a user-space context switch.  A raw `&tlsvar` is
+ * loop-invariant; inlined into malloc/free under LTO it can be hoisted across a
+ * swapcontext and reused on the OS thread a fiber migrated to -- a stale
+ * tsd/tcache.  Route all tsd access through this macro; never take `&tsd_tls`
+ * raw.  See https://github.com/jemalloc/jemalloc/issues/2890
+ *
+ * The accessor takes the address behind a noinline `memory` barrier, opaque to
+ * the optimizer.  MSVC has no inline asm, so it keeps the plain address; an MSVC
+ * build whose fibers run under /GL must instead compile with /GT (fiber-safe
+ * TLS).  DECLARE is emitted in every TU;
+ * DEFINE (the out-of-line body) only under JEMALLOC_TSD_C_, i.e. once in
+ * src/tsd.c.  Both must be invoked directly, not forwarded through another
+ * macro, or `##tlsvar` pastes the macro-expanded `je_tsd_tls` instead of the
+ * literal name.
+ */
+#if defined(__GNUC__)
+#  define JEMALLOC_TLS_ADDR_DECLARE(tlsvar)                                    \
+	__typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void);
+#  define JEMALLOC_TLS_ADDR_DEFINE(tlsvar)                                     \
+	JEMALLOC_NOINLINE __typeof__(&(tlsvar))                                       \
+	jemalloc_tls_addr_##tlsvar(void) {                                            \
+		__typeof__(&(tlsvar)) tls_addr = &(tlsvar);                                  \
+		__asm__ __volatile__("" : "+r"(tls_addr) : : "memory");                      \
+		return tls_addr;                                                             \
+	}
+#  define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar())
+#else
+#  define JEMALLOC_TLS_ADDR_DECLARE(tlsvar)
+#  define JEMALLOC_TLS_ADDR_DEFINE(tlsvar)
+#  define JEMALLOC_TLS_ADDR(tlsvar) (&(tlsvar))
+#endif
+
 /*
  * Thread-Specific-Data layout
  *

include/jemalloc/internal/tsd_malloc_thread_cleanup.h

@@ -13,6 +13,13 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls;
 extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized;
 extern bool tsd_booted;
 
+JEMALLOC_TLS_ADDR_DECLARE(tsd_tls)
+JEMALLOC_TLS_ADDR_DECLARE(tsd_initialized)
+#ifdef JEMALLOC_TSD_C_
+JEMALLOC_TLS_ADDR_DEFINE(tsd_tls)
+JEMALLOC_TLS_ADDR_DEFINE(tsd_initialized)
+#endif
+
 /* Initialization/cleanup. */
 JEMALLOC_ALWAYS_INLINE bool
 tsd_cleanup_wrapper(void) {
@@ -53,13 +60,15 @@ tsd_get_allocates(void) {
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
-	return &tsd_tls;
+	return JEMALLOC_TLS_ADDR(tsd_tls);
 }
 JEMALLOC_ALWAYS_INLINE void
 tsd_set(tsd_t *val) {
+	tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls);
+
 	assert(tsd_booted);
-	if (likely(&tsd_tls != val)) {
-		tsd_tls = (*val);
+	if (likely(tsd != val)) {
+		*tsd = (*val);
 	}
-	tsd_initialized = true;
+	*JEMALLOC_TLS_ADDR(tsd_initialized) = true;
 }

include/jemalloc/internal/tsd_tls.h

@@ -13,6 +13,11 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls;
 extern pthread_key_t tsd_tsd;
 extern bool          tsd_booted;
 
+JEMALLOC_TLS_ADDR_DECLARE(tsd_tls)
+#ifdef JEMALLOC_TSD_C_
+JEMALLOC_TLS_ADDR_DEFINE(tsd_tls)
+#endif
+
 /* Initialization/cleanup. */
 JEMALLOC_ALWAYS_INLINE bool
 tsd_boot0(void) {
@@ -46,16 +51,18 @@ tsd_get_allocates(void) {
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
-	return &tsd_tls;
+	return JEMALLOC_TLS_ADDR(tsd_tls);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 tsd_set(tsd_t *val) {
+	tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls);
+
 	assert(tsd_booted);
-	if (likely(&tsd_tls != val)) {
-		tsd_tls = (*val);
+	if (likely(tsd != val)) {
+		*tsd = (*val);
 	}
-	if (pthread_setspecific(tsd_tsd, (void *)(&tsd_tls)) != 0) {
+	if (pthread_setspecific(tsd_tsd, (void *)tsd) != 0) {
 		malloc_write("<jemalloc>: Error setting tsd.\n");
 		if (opt_abort) {
 			abort();

include/jemalloc/internal/tsd_win.h

@@ -178,6 +178,11 @@ tsd_set(tsd_t *val) {
 extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls;
 extern bool tsd_booted;
 
+JEMALLOC_TLS_ADDR_DECLARE(tsd_wrapper_tls)
+#ifdef JEMALLOC_TSD_C_
+JEMALLOC_TLS_ADDR_DEFINE(tsd_wrapper_tls)
+#endif
+
 /* Initialization/cleanup. */
 JEMALLOC_ALWAYS_INLINE bool
 tsd_cleanup_wrapper(void) {
@@ -223,16 +228,18 @@ tsd_get_allocates(void) {
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
-	return &(tsd_wrapper_tls.val);
+	return &(JEMALLOC_TLS_ADDR(tsd_wrapper_tls)->val);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 tsd_set(tsd_t *val) {
+	tsd_wrapper_t *wrapper = JEMALLOC_TLS_ADDR(tsd_wrapper_tls);
+
 	assert(tsd_booted);
-	if (likely(&(tsd_wrapper_tls.val) != val)) {
-		tsd_wrapper_tls.val = (*val);
+	if (likely(&wrapper->val != val)) {
+		wrapper->val = (*val);
 	}
-	tsd_wrapper_tls.initialized = true;
+	wrapper->initialized = true;
 }
 
 #endif // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER)

src/tsd.c

@@ -1,3 +1,6 @@
+/* Emit the single out-of-line JEMALLOC_TLS_ADDR offset globals/helpers here. */
+#define JEMALLOC_TSD_C_
+
 #include "jemalloc/internal/jemalloc_preamble.h"
 
 #include "jemalloc/internal/arenas_management.h"