diff --git a/configure.ac b/configure.ac index 12af2e99..9dcfd8f4 100644 --- a/configure.ac +++ b/configure.ac @@ -2841,6 +2841,8 @@ if test "x${je_cv_tls_model}" = "xyes" -a \ AC_DEFINE([JEMALLOC_TLS_MODEL], [__attribute__((tls_model("initial-exec")))], [ ]) + AC_DEFINE([JEMALLOC_TLS_MODEL_INITIAL_EXEC], [ ], + [Defined when the TSD thread-locals use the initial-exec (static) TLS model, i.e. live at a fixed offset from the thread pointer.]) else AC_DEFINE([JEMALLOC_TLS_MODEL], [ ], [ ]) fi diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 0b750a99..231a0d75 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -148,6 +148,13 @@ /* Non-empty if the tls_model attribute is supported. */ #undef JEMALLOC_TLS_MODEL +/* + * Defined when the TSD thread-locals use the initial-exec (static) TLS model, + * i.e. live at a fixed offset from the thread pointer. Gates the fast path of + * JEMALLOC_TLS_ADDR (see tsd_internals.h). + */ +#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC + /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables * inline functions. diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index a2027966..34af60f4 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -28,21 +28,78 @@ typedef struct prof_tdata_s prof_tdata_t; /* * JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler * cannot cache it across a user-space context switch. A raw `&tlsvar` is - * loop-invariant; inlined into malloc/free under LTO it can be hoisted across a - * swapcontext and reused on the OS thread a fiber migrated to -- a stale - * tsd/tcache. Route all tsd access through this macro; never take `&tsd_tls` - * raw. See https://github.com/jemalloc/jemalloc/issues/2890 + * `thread_pointer + const_offset`, loop-invariant; inlined into malloc/free + * under LTO it can be hoisted across a swapcontext and reused on the OS thread a + * fiber migrated to -- a stale tsd/tcache. Route all tsd access through this + * macro; never take `&tsd_tls` raw. See + * https://github.com/jemalloc/jemalloc/issues/2890 * - * The accessor takes the address behind a noinline `memory` barrier, opaque to - * the optimizer. MSVC has no inline asm, so it keeps the plain address; an MSVC - * build whose fibers run under /GL must instead compile with /GT (fiber-safe - * TLS). DECLARE is emitted in every TU; - * DEFINE (the out-of-line body) only under JEMALLOC_TSD_C_, i.e. once in - * src/tsd.c. Both must be invoked directly, not forwarded through another - * macro, or `##tlsvar` pastes the macro-expanded `je_tsd_tls` instead of the - * literal name. + * Static-TLS fast path (gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC): re-read the + * thread pointer with a volatile asm and add a runtime-captured constant offset. + * The offset MUST be captured by the noinline helper, not computed inline as + * `&tlsvar - thread_pointer` -- inline, the two terms hoist independently and + * cancel the volatile read back to the stale address. The offset is read and + * written with relaxed atomics -- threads racing their first allocation init it + * concurrently with the same thread-independent value. Other configs use a + * noinline `memory`-barrier accessor. MSVC has no inline asm, so it keeps the + * plain address; an MSVC build whose fibers run under whole-program opt (/GL) + * must instead compile with /GT (fiber-safe TLS). + * + * DECLARE is emitted in every TU; DEFINE (the out-of-line bodies) only under + * JEMALLOC_TSD_C_, i.e. once in src/tsd.c. Both must be invoked directly, not + * forwarded through another macro, or `##tlsvar` pastes the macro-expanded + * `je_tsd_tls` instead of the literal name. */ -#if defined(__GNUC__) +#if defined(__GNUC__) && !defined(_WIN32) && \ + defined(JEMALLOC_TLS_MODEL_INITIAL_EXEC) && \ + (defined(__aarch64__) || defined(__arm__) || defined(__x86_64__) || \ + defined(__i386__)) +JEMALLOC_ALWAYS_INLINE char * +jemalloc_thread_pointer(void) { + char *thread_pointer; +# if defined(__aarch64__) && defined(__APPLE__) + __asm__ __volatile__("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r"(thread_pointer)); +# elif defined(__aarch64__) + __asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(thread_pointer)); +# elif defined(__arm__) + __asm__ __volatile__("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r"(thread_pointer)); +# elif defined(__x86_64__) && defined(__APPLE__) + __asm__ __volatile__("movq %%gs:0, %0" : "=r"(thread_pointer)); +# elif defined(__x86_64__) + __asm__ __volatile__("movq %%fs:0, %0" : "=r"(thread_pointer)); +# else /* __i386__ */ + __asm__ __volatile__("movl %%gs:0, %0" : "=r"(thread_pointer)); +# endif + return thread_pointer; +} +/* 1 is unreachable: tlsvar and the thread pointer are at least 4-aligned. */ +# define JEMALLOC_TLS_OFFSET_UNINITIALIZED 1 +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ + extern intptr_t jemalloc_tls_offset_##tlsvar; \ + intptr_t jemalloc_tls_offset_init_##tlsvar(void); \ + JEMALLOC_ALWAYS_INLINE __typeof__(&(tlsvar)) \ + jemalloc_tls_addr_##tlsvar(void) { \ + intptr_t tls_offset = __atomic_load_n(&jemalloc_tls_offset_##tlsvar, \ + __ATOMIC_RELAXED); \ + if (unlikely(tls_offset == JEMALLOC_TLS_OFFSET_UNINITIALIZED)) { \ + tls_offset = jemalloc_tls_offset_init_##tlsvar(); \ + } \ + return (__typeof__(&(tlsvar)))(jemalloc_thread_pointer() + \ + tls_offset); \ + } +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \ + intptr_t jemalloc_tls_offset_##tlsvar = \ + JEMALLOC_TLS_OFFSET_UNINITIALIZED; \ + JEMALLOC_NOINLINE intptr_t \ + jemalloc_tls_offset_init_##tlsvar(void) { \ + intptr_t tls_offset = (intptr_t)((char *)&(tlsvar) - \ + jemalloc_thread_pointer()); \ + __atomic_store_n(&jemalloc_tls_offset_##tlsvar, tls_offset, \ + __ATOMIC_RELAXED); \ + return tls_offset; \ + } +# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar()) +#elif defined(__GNUC__) # define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ __typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void); # define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \