From 192271fb82214e34c522ec02b697bac5e136bfcf Mon Sep 17 00:00:00 2001 From: guangli-dai Date: Mon, 27 Jul 2026 17:08:58 -0700 Subject: [PATCH] Move sync (mutex) + cond + sigmask to OS layer Migrates the mutex backend selection out of mutex.h/mutex.c, and the background thread's condition-variable and signal-masking code out of background_thread.c, onto the OS layer dispatcher pattern. --- include/jemalloc/internal/background_thread.h | 5 +- include/jemalloc/internal/mutex.h | 73 +++--------------- include/jemalloc/internal/os.h | 9 +++ include/jemalloc/internal/os/cond.h | 38 ++++++++++ include/jemalloc/internal/os/darwin/mutex.h | 41 ++++++++++ include/jemalloc/internal/os/mutex.h | 39 ++++++++++ include/jemalloc/internal/os/posix/cond.h | 75 +++++++++++++++++++ include/jemalloc/internal/os/posix/mutex.h | 46 ++++++++++++ include/jemalloc/internal/os/posix/sigmask.h | 25 +++++++ include/jemalloc/internal/os/sigmask.h | 29 +++++++ include/jemalloc/internal/os/windows/mutex.h | 72 ++++++++++++++++++ src/background_thread.c | 67 +++++------------ src/jemalloc_init.c | 19 +++-- src/mutex.c | 37 +++------ 14 files changed, 431 insertions(+), 144 deletions(-) create mode 100644 include/jemalloc/internal/os/cond.h create mode 100644 include/jemalloc/internal/os/darwin/mutex.h create mode 100644 include/jemalloc/internal/os/mutex.h create mode 100644 include/jemalloc/internal/os/posix/cond.h create mode 100644 include/jemalloc/internal/os/posix/mutex.h create mode 100644 include/jemalloc/internal/os/posix/sigmask.h create mode 100644 include/jemalloc/internal/os/sigmask.h create mode 100644 include/jemalloc/internal/os/windows/mutex.h diff --git a/include/jemalloc/internal/background_thread.h b/include/jemalloc/internal/background_thread.h index c5096453..a423ab82 100644 --- a/include/jemalloc/internal/background_thread.h +++ b/include/jemalloc/internal/background_thread.h @@ -4,6 +4,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/base.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/os.h" #if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK) # define JEMALLOC_PTHREAD_CREATE_WRAPPER @@ -23,8 +24,8 @@ typedef enum { struct background_thread_info_s { #ifdef JEMALLOC_BACKGROUND_THREAD /* Background thread is pthread specific. */ - pthread_t thread; - pthread_cond_t cond; + pthread_t thread; + os_cond_t cond; #endif malloc_mutex_t mtx; background_thread_state_t state; diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index 943c7928..4ef90988 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -4,6 +4,7 @@ #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex_prof.h" +#include "jemalloc/internal/os.h" #include "jemalloc/internal/tsd.h" #include "jemalloc/internal/witness.h" @@ -40,19 +41,9 @@ struct malloc_mutex_s { * before release), and may be read by other threads. */ atomic_b_t locked; -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 - SRWLOCK lock; -# else - CRITICAL_SECTION lock; -# endif -#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - os_unfair_lock lock; -#elif (defined(JEMALLOC_MUTEX_INIT_CB)) - pthread_mutex_t lock; + os_mutex_t lock; +#ifdef JEMALLOC_MUTEX_INIT_CB malloc_mutex_t *postponed_next; -#else - pthread_mutex_t lock; #endif }; /* @@ -73,28 +64,9 @@ struct malloc_mutex_s { #endif }; -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 -# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) \ - ReleaseSRWLockExclusive(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) \ - (!TryAcquireSRWLockExclusive(&(m)->lock)) -# else -# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) \ - (!TryEnterCriticalSection(&(m)->lock)) -# endif -#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) -#else -# define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0) -#endif +#define MALLOC_MUTEX_LOCK(m) os_mutex_lock(&(m)->lock) +#define MALLOC_MUTEX_UNLOCK(m) os_mutex_unlock(&(m)->lock) +#define MALLOC_MUTEX_TRYLOCK(m) os_mutex_trylock(&(m)->lock) #define LOCK_PROF_DATA_INITIALIZER \ { \ @@ -102,34 +74,15 @@ struct malloc_mutex_s { ATOMIC_INIT(0), 0, NULL, 0 \ } -#ifdef _WIN32 +#if !OS_MUTEX_HAS_STATIC_INIT # define MALLOC_MUTEX_INITIALIZER -#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) +#elif defined(JEMALLOC_MUTEX_INIT_CB) # if defined(JEMALLOC_DEBUG) -# define MALLOC_MUTEX_INITIALIZER \ - { \ - {{LOCK_PROF_DATA_INITIALIZER, \ - ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \ - WITNESS_INITIALIZER( \ - "mutex", WITNESS_RANK_OMIT), \ - 0 \ - } -# else -# define MALLOC_MUTEX_INITIALIZER \ - { \ - {{LOCK_PROF_DATA_INITIALIZER, \ - ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \ - WITNESS_INITIALIZER( \ - "mutex", WITNESS_RANK_OMIT) \ - } -# endif -#elif (defined(JEMALLOC_MUTEX_INIT_CB)) -# if (defined(JEMALLOC_DEBUG)) # define MALLOC_MUTEX_INITIALIZER \ { \ {{LOCK_PROF_DATA_INITIALIZER, \ ATOMIC_INIT(false), \ - PTHREAD_MUTEX_INITIALIZER, NULL}}, \ + OS_MUTEX_INITIALIZER, NULL}}, \ WITNESS_INITIALIZER( \ "mutex", WITNESS_RANK_OMIT), \ 0 \ @@ -139,20 +92,18 @@ struct malloc_mutex_s { { \ {{LOCK_PROF_DATA_INITIALIZER, \ ATOMIC_INIT(false), \ - PTHREAD_MUTEX_INITIALIZER, NULL}}, \ + OS_MUTEX_INITIALIZER, NULL}}, \ WITNESS_INITIALIZER( \ "mutex", WITNESS_RANK_OMIT) \ } # endif - #else -# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT # if defined(JEMALLOC_DEBUG) # define MALLOC_MUTEX_INITIALIZER \ { \ {{LOCK_PROF_DATA_INITIALIZER, \ ATOMIC_INIT(false), \ - PTHREAD_MUTEX_INITIALIZER}}, \ + OS_MUTEX_INITIALIZER}}, \ WITNESS_INITIALIZER( \ "mutex", WITNESS_RANK_OMIT), \ 0 \ @@ -162,7 +113,7 @@ struct malloc_mutex_s { { \ {{LOCK_PROF_DATA_INITIALIZER, \ ATOMIC_INIT(false), \ - PTHREAD_MUTEX_INITIALIZER}}, \ + OS_MUTEX_INITIALIZER}}, \ WITNESS_INITIALIZER( \ "mutex", WITNESS_RANK_OMIT) \ } diff --git a/include/jemalloc/internal/os.h b/include/jemalloc/internal/os.h index 325dbbcd..99185907 100644 --- a/include/jemalloc/internal/os.h +++ b/include/jemalloc/internal/os.h @@ -34,4 +34,13 @@ /* Time */ #include "jemalloc/internal/os/time.h" +/* Sync (mutex) */ +#include "jemalloc/internal/os/mutex.h" + +/* Cond (POSIX background thread only) */ +#include "jemalloc/internal/os/cond.h" + +/* Sigmask (POSIX background thread only) */ +#include "jemalloc/internal/os/sigmask.h" + #endif /* JEMALLOC_INTERNAL_OS_H */ diff --git a/include/jemalloc/internal/os/cond.h b/include/jemalloc/internal/os/cond.h new file mode 100644 index 00000000..682104ff --- /dev/null +++ b/include/jemalloc/internal/os/cond.h @@ -0,0 +1,38 @@ +#ifndef JEMALLOC_INTERNAL_OS_COND_H +#define JEMALLOC_INTERNAL_OS_COND_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/os/detect.h" +#include "jemalloc/internal/os/mutex.h" + +/* + * Condition-variable interface. Exists only when background threads are + * supported (JEMALLOC_BACKGROUND_THREAD, pthread-only, never defined on + * Windows, and also never defined on Darwin despite Darwin being POSIX, per + * configure.ac's os_unfair_lock/macho exclusion): this header is a no-op on + * any platform that doesn't need it, so os.h can include it unconditionally + * without forcing a backend to exist where none is required. Only once + * background threads ARE needed does the inner POSIX-vs-else split apply, + * enforcing a real backend (os//cond.h) for any such platform. + * + * As with os/mutex.h, the enforced function prototypes come AFTER the + * backend #include below: os_cond_t is a type (embedded by value in + * background_thread_info_t), not just a function parameter. + */ +#if defined(JEMALLOC_BACKGROUND_THREAD) +# if defined(JEMALLOC_OS_POSIX) +# include "jemalloc/internal/os/posix/cond.h" +# else +# error "OS layer: no cond backend for this platform; add os//cond.h" +# endif + +/* Functions required for implementation in each backend. */ +JEMALLOC_ALWAYS_INLINE bool os_cond_init(os_cond_t *c); +JEMALLOC_ALWAYS_INLINE int os_cond_wait(os_cond_t *c, os_mutex_t *m); +JEMALLOC_ALWAYS_INLINE int os_cond_timedwait( + os_cond_t *c, os_mutex_t *m, const struct timespec *abs_ts); +JEMALLOC_ALWAYS_INLINE void os_cond_signal(os_cond_t *c); +JEMALLOC_ALWAYS_INLINE void os_cond_now(struct timespec *now); +#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */ + +#endif /* JEMALLOC_INTERNAL_OS_COND_H */ diff --git a/include/jemalloc/internal/os/darwin/mutex.h b/include/jemalloc/internal/os/darwin/mutex.h new file mode 100644 index 00000000..520442dc --- /dev/null +++ b/include/jemalloc/internal/os/darwin/mutex.h @@ -0,0 +1,41 @@ +#ifndef JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H +#define JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H + +/* + * Darwin os_unfair_lock backend (macOS 10.12+, selected via + * JEMALLOC_OS_UNFAIR_LOCK at configure time). os_unfair_lock has no destroy + * and no postfork_child fixup beyond resetting to OS_UNFAIR_LOCK_INIT (it's + * a single atomic word). Older macOS without os_unfair_lock falls back to + * os/posix/mutex.h's pthread backend instead of this file (see the + * dispatcher in os/mutex.h). + */ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include + +typedef os_unfair_lock os_mutex_t; +#define OS_MUTEX_INITIALIZER OS_UNFAIR_LOCK_INIT +#define OS_MUTEX_HAS_STATIC_INIT 1 + +JEMALLOC_ALWAYS_INLINE void +os_mutex_lock(os_mutex_t *m) { + os_unfair_lock_lock(m); +} + +JEMALLOC_ALWAYS_INLINE void +os_mutex_unlock(os_mutex_t *m) { + os_unfair_lock_unlock(m); +} + +JEMALLOC_ALWAYS_INLINE bool +os_mutex_trylock(os_mutex_t *m) { + return !os_unfair_lock_trylock(m); +} + +JEMALLOC_ALWAYS_INLINE bool +os_mutex_init(os_mutex_t *m) { + *m = (os_unfair_lock)OS_UNFAIR_LOCK_INIT; + return false; +} + +#endif /* JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H */ diff --git a/include/jemalloc/internal/os/mutex.h b/include/jemalloc/internal/os/mutex.h new file mode 100644 index 00000000..6d659d33 --- /dev/null +++ b/include/jemalloc/internal/os/mutex.h @@ -0,0 +1,39 @@ +#ifndef JEMALLOC_INTERNAL_OS_MUTEX_H +#define JEMALLOC_INTERNAL_OS_MUTEX_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/os/detect.h" + +/* + * Sync (mutex) interface. + * Default: posix/. Override: Darwin (os_unfair_lock), Windows (SRWLOCK / + * CRITICAL_SECTION). + * + * Unlike the other os/.h dispatchers, the enforced function + * prototypes come AFTER the backend #include below, not before: os_mutex_t + * is a type (embedded by value in malloc_mutex_t), not just a function + * parameter, so it must be fully defined by the backend before it can + * appear in a prototype at all. + * + * Capability macro: OS_MUTEX_HAS_STATIC_INIT: 1 iff OS_MUTEX_INITIALIZER is + * a valid static initializer for os_mutex_t, 0 if the mutex can only be + * initialized dynamically (via os_mutex_init()). Defined by whichever + * backend is selected below. + */ +#if defined(_WIN32) +# include "jemalloc/internal/os/windows/mutex.h" +#elif defined(JEMALLOC_OS_UNFAIR_LOCK) +# include "jemalloc/internal/os/darwin/mutex.h" +#elif defined(JEMALLOC_OS_POSIX) +# include "jemalloc/internal/os/posix/mutex.h" +#else +# error "OS layer: no mutex backend for this platform; add os//mutex.h" +#endif + +/* Functions required for implementation in each backend. */ +JEMALLOC_ALWAYS_INLINE void os_mutex_lock(os_mutex_t *m); +JEMALLOC_ALWAYS_INLINE void os_mutex_unlock(os_mutex_t *m); +JEMALLOC_ALWAYS_INLINE bool os_mutex_trylock(os_mutex_t *m); +JEMALLOC_ALWAYS_INLINE bool os_mutex_init(os_mutex_t *m); + +#endif /* JEMALLOC_INTERNAL_OS_MUTEX_H */ diff --git a/include/jemalloc/internal/os/posix/cond.h b/include/jemalloc/internal/os/posix/cond.h new file mode 100644 index 00000000..9b3ee755 --- /dev/null +++ b/include/jemalloc/internal/os/posix/cond.h @@ -0,0 +1,75 @@ +#ifndef JEMALLOC_INTERNAL_OS_POSIX_COND_H +#define JEMALLOC_INTERNAL_OS_POSIX_COND_H + +/* + * POSIX pthread condvar backend. Solely backs the background thread. + * + * No os_cond_destroy: the background thread is the only cond user and never + * destroys its condvars (it never did, even pre-refactor), so it would be + * dead code. Add one here if a caller ever needs it. + */ +#include "jemalloc/internal/jemalloc_preamble.h" + +typedef pthread_cond_t os_cond_t; + +JEMALLOC_ALWAYS_INLINE bool +os_cond_init(os_cond_t *c) { +#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC + /* + * Pair the condvar with CLOCK_MONOTONIC so os_cond_timedwait deadlines + * built from os_cond_now() are immune to wall-clock jumps. + */ + pthread_condattr_t attr; + if (pthread_condattr_init(&attr) != 0) { + return true; + } + if (pthread_condattr_setclock(&attr, CLOCK_MONOTONIC) != 0) { + pthread_condattr_destroy(&attr); + return true; + } + bool err = (pthread_cond_init(c, &attr) != 0); + pthread_condattr_destroy(&attr); + return err; +#else + return pthread_cond_init(c, NULL) != 0; +#endif +} + +JEMALLOC_ALWAYS_INLINE int +os_cond_wait(os_cond_t *c, os_mutex_t *m) { + return pthread_cond_wait(c, m); +} + +/* + * abs_ts is an absolute deadline in os_cond_now()'s clock (the clock the + * condvar was paired with in os_cond_init). Returns 0 on signal, ETIMEDOUT + * on timeout, other errno otherwise. + */ +JEMALLOC_ALWAYS_INLINE int +os_cond_timedwait(os_cond_t *c, os_mutex_t *m, const struct timespec *abs_ts) { + return pthread_cond_timedwait(c, m, abs_ts); +} + +JEMALLOC_ALWAYS_INLINE void +os_cond_signal(os_cond_t *c) { + pthread_cond_signal(c); +} + +/* + * Read "now" in the same clock os_cond_init() paired the condvar with, for + * building os_cond_timedwait() deadlines: CLOCK_MONOTONIC when available, + * else the wall clock. + */ +JEMALLOC_ALWAYS_INLINE void +os_cond_now(struct timespec *now) { +#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC + clock_gettime(CLOCK_MONOTONIC, now); +#else + struct timeval tv; + gettimeofday(&tv, NULL); + now->tv_sec = tv.tv_sec; + now->tv_nsec = tv.tv_usec * 1000; +#endif +} + +#endif /* JEMALLOC_INTERNAL_OS_POSIX_COND_H */ diff --git a/include/jemalloc/internal/os/posix/mutex.h b/include/jemalloc/internal/os/posix/mutex.h new file mode 100644 index 00000000..ce3acaf9 --- /dev/null +++ b/include/jemalloc/internal/os/posix/mutex.h @@ -0,0 +1,46 @@ +#ifndef JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H +#define JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H + +/* + * POSIX pthread mutex backend. Default for every POSIX platform, including + * Darwin builds without os_unfair_lock (older macOS). + */ +#include "jemalloc/internal/jemalloc_preamble.h" + +typedef pthread_mutex_t os_mutex_t; +#define OS_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +#define OS_MUTEX_HAS_STATIC_INIT 1 + +JEMALLOC_ALWAYS_INLINE void +os_mutex_lock(os_mutex_t *m) { + pthread_mutex_lock(m); +} + +JEMALLOC_ALWAYS_INLINE void +os_mutex_unlock(os_mutex_t *m) { + pthread_mutex_unlock(m); +} + +/* Returns true on failure (matches MALLOC_MUTEX_TRYLOCK semantics). */ +JEMALLOC_ALWAYS_INLINE bool +os_mutex_trylock(os_mutex_t *m) { + return pthread_mutex_trylock(m) != 0; +} + +/* Returns true on failure. */ +JEMALLOC_ALWAYS_INLINE bool +os_mutex_init(os_mutex_t *m) { + pthread_mutexattr_t attr; + if (pthread_mutexattr_init(&attr) != 0) { + return true; + } + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); + if (pthread_mutex_init(m, &attr) != 0) { + pthread_mutexattr_destroy(&attr); + return true; + } + pthread_mutexattr_destroy(&attr); + return false; +} + +#endif /* JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H */ diff --git a/include/jemalloc/internal/os/posix/sigmask.h b/include/jemalloc/internal/os/posix/sigmask.h new file mode 100644 index 00000000..44d7c5e0 --- /dev/null +++ b/include/jemalloc/internal/os/posix/sigmask.h @@ -0,0 +1,25 @@ +#ifndef JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H +#define JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H + +/* + * POSIX sigset_t-based signal-mask backend. Solely backs the background + * thread's pthread_create wrapper. + */ +#include "jemalloc/internal/jemalloc_preamble.h" + +typedef sigset_t os_sigmask_t; + +/* Mask all signals, returning the prior mask in *saved. */ +JEMALLOC_ALWAYS_INLINE int +os_sigmask_all_enter(os_sigmask_t *saved) { + sigset_t set; + sigfillset(&set); + return pthread_sigmask(SIG_SETMASK, &set, saved); +} + +JEMALLOC_ALWAYS_INLINE int +os_sigmask_leave(const os_sigmask_t *saved) { + return pthread_sigmask(SIG_SETMASK, saved, NULL); +} + +#endif /* JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H */ diff --git a/include/jemalloc/internal/os/sigmask.h b/include/jemalloc/internal/os/sigmask.h new file mode 100644 index 00000000..c3e39f99 --- /dev/null +++ b/include/jemalloc/internal/os/sigmask.h @@ -0,0 +1,29 @@ +#ifndef JEMALLOC_INTERNAL_OS_SIGMASK_H +#define JEMALLOC_INTERNAL_OS_SIGMASK_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/os/detect.h" + +/* + * Signal-mask interface. Exists only when background threads are supported + * (JEMALLOC_BACKGROUND_THREAD), for the same reason as os/cond.h: it backs + * the background thread (masking signals during pthread_create so the new + * thread inherits an empty signal set), which is pthread-specific and never + * built on Windows -- and never enabled on Darwin either, despite Darwin + * being POSIX. Gating on JEMALLOC_BACKGROUND_THREAD first, rather than + * JEMALLOC_OS_POSIX alone, means this header is a no-op wherever background + * threads aren't needed, so os.h can include it unconditionally. + */ +#if defined(JEMALLOC_BACKGROUND_THREAD) +# if defined(JEMALLOC_OS_POSIX) +# include "jemalloc/internal/os/posix/sigmask.h" +# else +# error "OS layer: no sigmask backend for this platform; add os//sigmask.h" +# endif + +/* Functions required for implementation in each backend. */ +JEMALLOC_ALWAYS_INLINE int os_sigmask_all_enter(os_sigmask_t *saved); +JEMALLOC_ALWAYS_INLINE int os_sigmask_leave(const os_sigmask_t *saved); +#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */ + +#endif /* JEMALLOC_INTERNAL_OS_SIGMASK_H */ diff --git a/include/jemalloc/internal/os/windows/mutex.h b/include/jemalloc/internal/os/windows/mutex.h new file mode 100644 index 00000000..32dd35c4 --- /dev/null +++ b/include/jemalloc/internal/os/windows/mutex.h @@ -0,0 +1,72 @@ +#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H +#define JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H + +/* + * Windows mutex backend. On Vista+, os_mutex_t is SRWLOCK (lighter than + * CRITICAL_SECTION); older targets fall back to CRITICAL_SECTION with a + * spin count. + */ +#include "jemalloc/internal/jemalloc_preamble.h" + +#if _WIN32_WINNT >= 0x0600 +typedef SRWLOCK os_mutex_t; +# define OS_MUTEX_INITIALIZER SRWLOCK_INIT +# define OS_MUTEX_HAS_STATIC_INIT 1 + +JEMALLOC_ALWAYS_INLINE void +os_mutex_lock(os_mutex_t *m) { + AcquireSRWLockExclusive(m); +} + +JEMALLOC_ALWAYS_INLINE void +os_mutex_unlock(os_mutex_t *m) { + ReleaseSRWLockExclusive(m); +} + +JEMALLOC_ALWAYS_INLINE bool +os_mutex_trylock(os_mutex_t *m) { + return !TryAcquireSRWLockExclusive(m); +} + +JEMALLOC_ALWAYS_INLINE bool +os_mutex_init(os_mutex_t *m) { + InitializeSRWLock(m); + return false; +} +#else +typedef CRITICAL_SECTION os_mutex_t; +/* + * CRITICAL_SECTION has no static initializer, unlike SRWLOCK above -- so + * MALLOC_MUTEX_INITIALIZER is empty here, and jemalloc_init.c's init_lock + * (the one static malloc_mutex_t that must be usable before any other code + * runs, on every _WIN32_WINNT target) still needs its own constructor-based + * lazy-init workaround on this pre-Vista path. + */ +# define OS_MUTEX_HAS_STATIC_INIT 0 + +# ifndef _CRT_SPINCOUNT +# define _CRT_SPINCOUNT 4000 +# endif + +JEMALLOC_ALWAYS_INLINE void +os_mutex_lock(os_mutex_t *m) { + EnterCriticalSection(m); +} + +JEMALLOC_ALWAYS_INLINE void +os_mutex_unlock(os_mutex_t *m) { + LeaveCriticalSection(m); +} + +JEMALLOC_ALWAYS_INLINE bool +os_mutex_trylock(os_mutex_t *m) { + return !TryEnterCriticalSection(m); +} + +JEMALLOC_ALWAYS_INLINE bool +os_mutex_init(os_mutex_t *m) { + return !InitializeCriticalSectionAndSpinCount(m, _CRT_SPINCOUNT); +} +#endif + +#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H */ diff --git a/src/background_thread.c b/src/background_thread.c index 179ed617..1fde54a2 100644 --- a/src/background_thread.c +++ b/src/background_thread.c @@ -10,6 +10,7 @@ #include "jemalloc/internal/jemalloc_internal_inlines_a.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/os.h" #include "jemalloc/internal/prof.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/witness.h" @@ -77,7 +78,7 @@ background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind) { assert(info->state == background_thread_paused); info->state = background_thread_started; #ifdef JEMALLOC_BACKGROUND_THREAD - pthread_cond_signal(&info->cond); + os_cond_signal(&info->cond); #endif malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); } @@ -247,59 +248,31 @@ background_thread_cond_wait( int ret; /* - * pthread_cond_wait drops and re-acquires the mutex internally, w/o - * going through our wrapper. Update the locked state explicitly. + * os_cond_wait drops and re-acquires the mutex internally, w/o going + * through our wrapper. Update the locked state explicitly. */ atomic_store_b(&info->mtx.locked, false, ATOMIC_RELAXED); if (ts == NULL) { - ret = pthread_cond_wait(&info->cond, &info->mtx.lock); + ret = os_cond_wait(&info->cond, &info->mtx.lock); } else { - ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, ts); + ret = os_cond_timedwait(&info->cond, &info->mtx.lock, ts); } atomic_store_b(&info->mtx.locked, true, ATOMIC_RELAXED); return ret; } -static int -background_thread_cond_init(pthread_cond_t *cond) { -#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC - pthread_condattr_t cond_attr; - int ret = pthread_condattr_init(&cond_attr); - if (ret != 0) { - return ret; - } - ret = pthread_condattr_setclock(&cond_attr, CLOCK_MONOTONIC); - if (ret != 0) { - pthread_condattr_destroy(&cond_attr); - return ret; - } - ret = pthread_cond_init(cond, &cond_attr); - pthread_condattr_destroy(&cond_attr); - return ret; -#else - return pthread_cond_init(cond, NULL); -#endif -} - /* - * Fill in the absolute deadline for pthread_cond_timedwait. The clock read - * here MUST match the clock the condvar was initialized with in - * background_thread_cond_init, otherwise the deadline is interpreted against - * the wrong epoch. + * Fill in the absolute deadline for os_cond_timedwait. os_cond_now() reads + * the same clock os_cond_init() paired the condvar with, so the deadline + * lands in the right epoch. */ static void background_thread_wakeup_ts_init(struct timespec *ts, uint64_t interval) { nstime_t wakeup; -#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC struct timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); + os_cond_now(&now); nstime_init2(&wakeup, now.tv_sec, now.tv_nsec); -#else - struct timeval tv; - gettimeofday(&tv, NULL); - nstime_init2(&wakeup, tv.tv_sec, tv.tv_usec * 1000); -#endif nstime_iadd(&wakeup, interval); ts->tv_sec = (size_t)nstime_sec(&wakeup); ts->tv_nsec = (size_t)nstime_nsec(&wakeup); @@ -426,7 +399,7 @@ background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) { if (info->state == background_thread_started) { has_thread = true; info->state = background_thread_stopped; - pthread_cond_signal(&info->cond); + os_cond_signal(&info->cond); } else { has_thread = false; } @@ -457,10 +430,8 @@ background_thread_create_signals_masked(pthread_t *thread, * Mask signals during thread creation so that the thread inherits * an empty signal set. */ - sigset_t set; - sigfillset(&set); - sigset_t oldset; - int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset); + os_sigmask_t oldset; + int mask_err = os_sigmask_all_enter(&oldset); if (mask_err != 0) { return mask_err; } @@ -470,7 +441,7 @@ background_thread_create_signals_masked(pthread_t *thread, * Restore the signal mask. Failure to restore the signal mask here * changes program behavior. */ - int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL); + int restore_err = os_sigmask_leave(&oldset); if (restore_err != 0) { malloc_printf( ": background thread creation " @@ -681,7 +652,7 @@ background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { /* Threads are created asynchronously by Thread 0. */ background_thread_info_t *t0 = &background_thread_info[0]; malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx); - pthread_cond_signal(&t0->cond); + os_cond_signal(&t0->cond); malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx); return false; @@ -813,7 +784,7 @@ background_thread_wakeup_early( && nstime_ns(remaining_sleep) < BACKGROUND_THREAD_MIN_INTERVAL_NS) { return; } - pthread_cond_signal(&info->cond); + os_cond_signal(&info->cond); } void @@ -857,8 +828,8 @@ background_thread_postfork_child(tsdn_t *tsdn) { background_thread_info_t *info = &background_thread_info[i]; malloc_mutex_lock(tsdn, &info->mtx); info->state = background_thread_stopped; - int ret = background_thread_cond_init(&info->cond); - assert(ret == 0); + bool ret = os_cond_init(&info->cond); + assert(!ret); background_thread_info_init(tsdn, info); malloc_mutex_unlock(tsdn, &info->mtx); } @@ -975,7 +946,7 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) { malloc_mutex_address_ordered)) { return true; } - if (background_thread_cond_init(&info->cond)) { + if (os_cond_init(&info->cond)) { return true; } malloc_mutex_lock(tsdn, &info->mtx); diff --git a/src/jemalloc_init.c b/src/jemalloc_init.c index 34cfdcf5..27ebc3fe 100644 --- a/src/jemalloc_init.c +++ b/src/jemalloc_init.c @@ -55,10 +55,14 @@ malloc_initializer_set(void) { } /* Used to avoid initialization races. */ -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 -static malloc_mutex_t init_lock = SRWLOCK_INIT; -# else +#if !OS_MUTEX_HAS_STATIC_INIT +/* + * This target's os_mutex_t has no static initializer (currently: pre-Vista + * Windows CRITICAL_SECTION; see os/windows/mutex.h), so MALLOC_MUTEX_ + * INITIALIZER is empty -- init_lock needs its own lazy-init workaround + * instead, being the one static malloc_mutex_t that must be usable before + * any other code runs. + */ static malloc_mutex_t init_lock; static bool init_lock_initialized = false; @@ -82,12 +86,11 @@ _init_init_lock(void) { init_lock_initialized = true; } -# ifdef _MSC_VER -# pragma section(".CRT$XCU", read) +# ifdef _MSC_VER +# pragma section(".CRT$XCU", read) JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) static const void(WINAPI *init_init_lock)(void) = _init_init_lock; -# endif # endif #else static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; @@ -599,7 +602,7 @@ malloc_init_hard(void) { */ assert(SC_NTINY == 0 || SC_LG_TINY_MAXCLASS <= SC_LG_LARGE_MINCLASS); -#if defined(_WIN32) && _WIN32_WINNT < 0x0600 +#if !OS_MUTEX_HAS_STATIC_INIT _init_init_lock(); #endif malloc_mutex_lock(TSDN_NULL, &init_lock); diff --git a/src/mutex.c b/src/mutex.c index 04a14a24..c31af503 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -5,10 +5,6 @@ #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/spin.h" -#if defined(_WIN32) && !defined(_CRT_SPINCOUNT) -# define _CRT_SPINCOUNT 4000 -#endif - /* * Based on benchmark results, a fixed spin with this amount of retries works * well for our critical sections. @@ -141,18 +137,17 @@ bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank, malloc_mutex_lock_order_t lock_order) { mutex_prof_data_init(&mutex->prof_data); -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 - InitializeSRWLock(&mutex->lock); -# else - if (!InitializeCriticalSectionAndSpinCount( - &mutex->lock, _CRT_SPINCOUNT)) { - return true; - } -# endif -#elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - mutex->lock = OS_UNFAIR_LOCK_INIT; -#elif (defined(JEMALLOC_MUTEX_INIT_CB)) +#ifdef JEMALLOC_MUTEX_INIT_CB + /* + * Targets platforms with _pthread_mutex_init_calloc_cb(), which is + * currently just FreeBSD/libthr: its pthread_mutex_init() can calloc() + * internally, which would recurse back into jemalloc before it is safe + * safe to allocate. Until malloc_mutex_boot() runs, defer real + * initialization by queuing the mutex here instead; malloc_mutex_boot() + * later drains the queue, calling _pthread_mutex_init_calloc_cb() with + * bootstrap_calloc (jemalloc's own early-bootstrap allocation path) as + * the callback, once that's safe to use. + */ if (postpone_init) { mutex->postponed_next = postponed_mutexes; postponed_mutexes = mutex; @@ -164,17 +159,9 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank, } } #else - pthread_mutexattr_t attr; - - if (pthread_mutexattr_init(&attr) != 0) { + if (os_mutex_init(&mutex->lock)) { return true; } - pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE); - if (pthread_mutex_init(&mutex->lock, &attr) != 0) { - pthread_mutexattr_destroy(&attr); - return true; - } - pthread_mutexattr_destroy(&attr); #endif if (config_debug) { mutex->lock_order = lock_order;