romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-18 14:47:18 +03:00
Code Issues Projects Releases Packages Wiki Activity

Add an inline fast path to JEMALLOC_TLS_ADDR for static TLS

Browse source 
The noinline accessor puts an opaque call on the malloc/free fastpath.
test/stress/microbench (malloc(1)/free pairs, pinned core, clang):

                     unpatched      accessor       this commit
  malloc, no LTO     7.0 ns/op      9.8 (+40%)     7.0 (+-0%)
  free,   no LTO     6.5 ns/op      9.2 (+40%)     6.5 (+-0%)
  malloc, ThinLTO    7.0 ns/op      9.0 (+28%)     7.6 (~+8%)
  free,   ThinLTO    6.5 ns/op      8.4 (+28%)     7.0 (~+8%)

Under a static TLS model the tsd address is `thread_pointer + offset` with a
thread-independent offset.  Capture the offset once (a noinline helper, lazy and
sentinel-initialized) and re-read the thread pointer per call with a `volatile`
asm the optimizer may not hoist or CSE (per-arch reads from mimalloc's
mi_prim_tls_slot).  Both inputs are hoist-proof: the tp read is volatile, the
offset is thread-independent.

The offset global is read and written with relaxed atomics: threads racing
their first allocation init it concurrently with the same value, so a plain
access would be a benign-but-UB data race (TSan-reported).  Relaxed adds no
fence -- it compiles to the same load/store on the fast path.

The offset must NOT be computed inline as `&tsd_tls - __builtin_thread_pointer()`:
the terms hoist independently, and clang ThinLTO keeps the stale `&tsd_tls` in a
callee-saved register while re-reading the thread pointer, so `fresh_tp +
(stale_addr - fresh_tp)` cancels back to the stale address.  Capturing behind a
call boundary evaluates both terms at one point on one thread.

Gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC (a new configure define, set whenever
jemalloc applies its default initial-exec model), GNU asm, a known arch, and
!_WIN32 (MinGW's thread pointer lives in the TEB, not fs/gs:0); everything else
keeps the noinline accessor.  Verified with the fiber-migration reproducer under
clang ThinLTO.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Azat Khuzhin 2026-06-22 12:47:46 +02:00
parent 7b9853c4a1
commit 7aa77c8e2a
3 changed files with 79 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

2
configure.ac
View file

@ -2841,6 +2841,8 @@ if test "x${je_cv_tls_model}" = "xyes" -a \
AC_DEFINE([JEMALLOC_TLS_MODEL],
[__attribute__((tls_model("initial-exec")))],
[ ])
AC_DEFINE([JEMALLOC_TLS_MODEL_INITIAL_EXEC], [ ],
[Defined when the TSD thread-locals use the initial-exec (static) TLS model, i.e. live at a fixed offset from the thread pointer.])
else
AC_DEFINE([JEMALLOC_TLS_MODEL], [ ], [ ])
fi

7
include/jemalloc/internal/jemalloc_internal_defs.h.in
View file

@ -148,6 +148,13 @@
/* Non-empty if the tls_model attribute is supported. */
#undef JEMALLOC_TLS_MODEL
/*
* Defined when the TSD thread-locals use the initial-exec (static) TLS model,
* i.e. live at a fixed offset from the thread pointer. Gates the fast path of
* JEMALLOC_TLS_ADDR (see tsd_internals.h).
*/
#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC
/*
* JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
* inline functions.

83
include/jemalloc/internal/tsd_internals.h
View file

@ -28,21 +28,78 @@ typedef struct prof_tdata_s prof_tdata_t;
/*
* JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler
* cannot cache it across a user-space context switch. A raw `&tlsvar` is
* loop-invariant; inlined into malloc/free under LTO it can be hoisted across a
* swapcontext and reused on the OS thread a fiber migrated to -- a stale
* tsd/tcache. Route all tsd access through this macro; never take `&tsd_tls`
* raw. See https://github.com/jemalloc/jemalloc/issues/2890
* `thread_pointer + const_offset`, loop-invariant; inlined into malloc/free
* under LTO it can be hoisted across a swapcontext and reused on the OS thread a
* fiber migrated to -- a stale tsd/tcache. Route all tsd access through this
* macro; never take `&tsd_tls` raw. See
* https://github.com/jemalloc/jemalloc/issues/2890
*
* The accessor takes the address behind a noinline `memory` barrier, opaque to
* the optimizer. MSVC has no inline asm, so it keeps the plain address; an MSVC
* build whose fibers run under /GL must instead compile with /GT (fiber-safe
* TLS). DECLARE is emitted in every TU;
* DEFINE (the out-of-line body) only under JEMALLOC_TSD_C_, i.e. once in
* src/tsd.c. Both must be invoked directly, not forwarded through another
* macro, or `##tlsvar` pastes the macro-expanded `je_tsd_tls` instead of the
* literal name.
* Static-TLS fast path (gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC): re-read the
* thread pointer with a volatile asm and add a runtime-captured constant offset.
* The offset MUST be captured by the noinline helper, not computed inline as
* `&tlsvar - thread_pointer` -- inline, the two terms hoist independently and
* cancel the volatile read back to the stale address. The offset is read and
* written with relaxed atomics -- threads racing their first allocation init it
* concurrently with the same thread-independent value. Other configs use a
* noinline `memory`-barrier accessor. MSVC has no inline asm, so it keeps the
* plain address; an MSVC build whose fibers run under whole-program opt (/GL)
* must instead compile with /GT (fiber-safe TLS).
*
* DECLARE is emitted in every TU; DEFINE (the out-of-line bodies) only under
* JEMALLOC_TSD_C_, i.e. once in src/tsd.c. Both must be invoked directly, not
* forwarded through another macro, or `##tlsvar` pastes the macro-expanded
* `je_tsd_tls` instead of the literal name.
*/
#if defined(__GNUC__)
#if defined(__GNUC__) && !defined(_WIN32) && \
defined(JEMALLOC_TLS_MODEL_INITIAL_EXEC) && \
(defined(__aarch64__) || defined(__arm__) || defined(__x86_64__) || \
defined(__i386__))
JEMALLOC_ALWAYS_INLINE char *
jemalloc_thread_pointer(void) {
char *thread_pointer;
# if defined(__aarch64__) && defined(__APPLE__)
__asm__ __volatile__("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r"(thread_pointer));
# elif defined(__aarch64__)
__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(thread_pointer));
# elif defined(__arm__)
__asm__ __volatile__("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r"(thread_pointer));
# elif defined(__x86_64__) && defined(__APPLE__)
__asm__ __volatile__("movq %%gs:0, %0" : "=r"(thread_pointer));
# elif defined(__x86_64__)
__asm__ __volatile__("movq %%fs:0, %0" : "=r"(thread_pointer));
# else /* __i386__ */
__asm__ __volatile__("movl %%gs:0, %0" : "=r"(thread_pointer));
# endif
return thread_pointer;
}
/* 1 is unreachable: tlsvar and the thread pointer are at least 4-aligned. */
# define JEMALLOC_TLS_OFFSET_UNINITIALIZED 1
# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \
extern intptr_t jemalloc_tls_offset_##tlsvar; \
intptr_t jemalloc_tls_offset_init_##tlsvar(void); \
JEMALLOC_ALWAYS_INLINE __typeof__(&(tlsvar)) \
jemalloc_tls_addr_##tlsvar(void) { \
intptr_t tls_offset = __atomic_load_n(&jemalloc_tls_offset_##tlsvar, \
__ATOMIC_RELAXED); \
if (unlikely(tls_offset == JEMALLOC_TLS_OFFSET_UNINITIALIZED)) { \
tls_offset = jemalloc_tls_offset_init_##tlsvar(); \
} \
return (__typeof__(&(tlsvar)))(jemalloc_thread_pointer() + \
tls_offset); \
}
# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \
intptr_t jemalloc_tls_offset_##tlsvar = \
JEMALLOC_TLS_OFFSET_UNINITIALIZED; \
JEMALLOC_NOINLINE intptr_t \
jemalloc_tls_offset_init_##tlsvar(void) { \
intptr_t tls_offset = (intptr_t)((char *)&(tlsvar) - \
jemalloc_thread_pointer()); \
__atomic_store_n(&jemalloc_tls_offset_##tlsvar, tls_offset, \
__ATOMIC_RELAXED); \
return tls_offset; \
}
# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar())
#elif defined(__GNUC__)
# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \
__typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void);
# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \