romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-22 08:37:18 +03:00
Code Issues Projects Releases Packages Wiki Activity

Merge 7aa77c8e2a into b44e84086f

Browse source
This commit is contained in:
Azat Khuzhin 2026-07-17 12:32:51 +08:00 • committed by GitHub
commit bf09e8b96c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 368 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

7
include/jemalloc/internal/jemalloc_internal_defs.h.in
View file

@ -151,6 +151,13 @@
/* Non-empty if the tls_model attribute is supported. */
#undef JEMALLOC_TLS_MODEL
/*
* Defined when the TSD thread-locals use the initial-exec (static) TLS model,
* i.e. live at a fixed offset from the thread pointer. Gates the fast path of
* JEMALLOC_TLS_ADDR (see tsd_internals.h).
*/
#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC
/*
* JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
* inline functions.

91
include/jemalloc/internal/tsd_internals.h
View file

@ -25,6 +25,97 @@
typedef struct arena_s arena_t;
typedef struct prof_tdata_s prof_tdata_t;
/*
* JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler
* cannot cache it across a user-space context switch. A raw `&tlsvar` is
* `thread_pointer + const_offset`, loop-invariant; inlined into malloc/free
* under LTO it can be hoisted across a swapcontext and reused on the OS thread a
* fiber migrated to -- a stale tsd/tcache. Route all tsd access through this
* macro; never take `&tsd_tls` raw. See
* https://github.com/jemalloc/jemalloc/issues/2890
*
* Static-TLS fast path (gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC): re-read the
* thread pointer with a volatile asm and add a runtime-captured constant offset.
* The offset MUST be captured by the noinline helper, not computed inline as
* `&tlsvar - thread_pointer` -- inline, the two terms hoist independently and
* cancel the volatile read back to the stale address. The offset is read and
* written with relaxed atomics -- threads racing their first allocation init it
* concurrently with the same thread-independent value. Other configs use a
* noinline `memory`-barrier accessor. MSVC has no inline asm, so it keeps the
* plain address; an MSVC build whose fibers run under whole-program opt (/GL)
* must instead compile with /GT (fiber-safe TLS).
*
* DECLARE is emitted in every TU; DEFINE (the out-of-line bodies) only under
* JEMALLOC_TSD_C_, i.e. once in src/tsd.c. Both must be invoked directly, not
* forwarded through another macro, or `##tlsvar` pastes the macro-expanded
* `je_tsd_tls` instead of the literal name.
*/
#if defined(__GNUC__) && !defined(_WIN32) && \
defined(JEMALLOC_TLS_MODEL_INITIAL_EXEC) && \
(defined(__aarch64__) || defined(__arm__) || defined(__x86_64__) || \
defined(__i386__))
JEMALLOC_ALWAYS_INLINE char *
jemalloc_thread_pointer(void) {
char *thread_pointer;
# if defined(__aarch64__) && defined(__APPLE__)
__asm__ __volatile__("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r"(thread_pointer));
# elif defined(__aarch64__)
__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(thread_pointer));
# elif defined(__arm__)
__asm__ __volatile__("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r"(thread_pointer));
# elif defined(__x86_64__) && defined(__APPLE__)
__asm__ __volatile__("movq %%gs:0, %0" : "=r"(thread_pointer));
# elif defined(__x86_64__)
__asm__ __volatile__("movq %%fs:0, %0" : "=r"(thread_pointer));
# else /* __i386__ */
__asm__ __volatile__("movl %%gs:0, %0" : "=r"(thread_pointer));
# endif
return thread_pointer;
}
/* 1 is unreachable: tlsvar and the thread pointer are at least 4-aligned. */
# define JEMALLOC_TLS_OFFSET_UNINITIALIZED 1
# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \
extern intptr_t jemalloc_tls_offset_##tlsvar; \
intptr_t jemalloc_tls_offset_init_##tlsvar(void); \
JEMALLOC_ALWAYS_INLINE __typeof__(&(tlsvar)) \
jemalloc_tls_addr_##tlsvar(void) { \
intptr_t tls_offset = __atomic_load_n(&jemalloc_tls_offset_##tlsvar, \
__ATOMIC_RELAXED); \
if (unlikely(tls_offset == JEMALLOC_TLS_OFFSET_UNINITIALIZED)) { \
tls_offset = jemalloc_tls_offset_init_##tlsvar(); \
} \
return (__typeof__(&(tlsvar)))(jemalloc_thread_pointer() + \
tls_offset); \
}
# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \
intptr_t jemalloc_tls_offset_##tlsvar = \
JEMALLOC_TLS_OFFSET_UNINITIALIZED; \
JEMALLOC_NOINLINE intptr_t \
jemalloc_tls_offset_init_##tlsvar(void) { \
intptr_t tls_offset = (intptr_t)((char *)&(tlsvar) - \
jemalloc_thread_pointer()); \
__atomic_store_n(&jemalloc_tls_offset_##tlsvar, tls_offset, \
__ATOMIC_RELAXED); \
return tls_offset; \
}
# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar())
#elif defined(__GNUC__)
# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \
__typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void);
# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \
JEMALLOC_NOINLINE __typeof__(&(tlsvar)) \
jemalloc_tls_addr_##tlsvar(void) { \
__typeof__(&(tlsvar)) tls_addr = &(tlsvar); \
__asm__ __volatile__("" : "+r"(tls_addr) : : "memory"); \
return tls_addr; \
}
# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar())
#else
# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar)
# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar)
# define JEMALLOC_TLS_ADDR(tlsvar) (&(tlsvar))
#endif
/*
* Thread-Specific-Data layout
*

17
include/jemalloc/internal/tsd_malloc_thread_cleanup.h
View file

@ -13,6 +13,13 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls;
extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized;
extern bool tsd_booted;
JEMALLOC_TLS_ADDR_DECLARE(tsd_tls)
JEMALLOC_TLS_ADDR_DECLARE(tsd_initialized)
#ifdef JEMALLOC_TSD_C_
JEMALLOC_TLS_ADDR_DEFINE(tsd_tls)
JEMALLOC_TLS_ADDR_DEFINE(tsd_initialized)
#endif
/* Initialization/cleanup. */
JEMALLOC_ALWAYS_INLINE bool
tsd_cleanup_wrapper(void) {
@ -58,13 +65,15 @@ tsd_teardown_done(void) {
/* Get/set. */
JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_get(bool init) {
return &tsd_tls;
return JEMALLOC_TLS_ADDR(tsd_tls);
}
JEMALLOC_ALWAYS_INLINE void
tsd_set(tsd_t *val) {
tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls);
assert(tsd_booted);
if (likely(&tsd_tls != val)) {
tsd_tls = (*val);
if (likely(tsd != val)) {
*tsd = (*val);
}
tsd_initialized = true;
*JEMALLOC_TLS_ADDR(tsd_initialized) = true;
}

15
include/jemalloc/internal/tsd_tls.h
View file

@ -13,6 +13,11 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls;
extern pthread_key_t tsd_tsd;
extern bool tsd_booted;
JEMALLOC_TLS_ADDR_DECLARE(tsd_tls)
#ifdef JEMALLOC_TSD_C_
JEMALLOC_TLS_ADDR_DEFINE(tsd_tls)
#endif
/* Initialization/cleanup. */
JEMALLOC_ALWAYS_INLINE bool
tsd_boot0(void) {
@ -51,16 +56,18 @@ tsd_teardown_done(void) {
/* Get/set. */
JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_get(bool init) {
return &tsd_tls;
return JEMALLOC_TLS_ADDR(tsd_tls);
}
JEMALLOC_ALWAYS_INLINE void
tsd_set(tsd_t *val) {
tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls);
assert(tsd_booted);
if (likely(&tsd_tls != val)) {
tsd_tls = (*val);
if (likely(tsd != val)) {
*tsd = (*val);
}
if (pthread_setspecific(tsd_tsd, (void *)(&tsd_tls)) != 0) {
if (pthread_setspecific(tsd_tsd, (void *)tsd) != 0) {
malloc_write("<jemalloc>: Error setting tsd.\n");
if (opt_abort) {
abort();

15
include/jemalloc/internal/tsd_win.h
View file

@ -183,6 +183,11 @@ tsd_set(tsd_t *val) {
extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls;
extern bool tsd_booted;
JEMALLOC_TLS_ADDR_DECLARE(tsd_wrapper_tls)
#ifdef JEMALLOC_TSD_C_
JEMALLOC_TLS_ADDR_DEFINE(tsd_wrapper_tls)
#endif
/* Initialization/cleanup. */
JEMALLOC_ALWAYS_INLINE bool
tsd_cleanup_wrapper(void) {
@ -233,16 +238,18 @@ tsd_teardown_done(void) {
/* Get/set. */
JEMALLOC_ALWAYS_INLINE tsd_t *
tsd_get(bool init) {
return &(tsd_wrapper_tls.val);
return &(JEMALLOC_TLS_ADDR(tsd_wrapper_tls)->val);
}
JEMALLOC_ALWAYS_INLINE void
tsd_set(tsd_t *val) {
tsd_wrapper_t *wrapper = JEMALLOC_TLS_ADDR(tsd_wrapper_tls);
assert(tsd_booted);
if (likely(&(tsd_wrapper_tls.val) != val)) {
tsd_wrapper_tls.val = (*val);
if (likely(&wrapper->val != val)) {
wrapper->val = (*val);
}
tsd_wrapper_tls.initialized = true;
wrapper->initialized = true;
}
#endif // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER)