romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-08-05 01:46:16 +03:00
Code Issues Projects Releases Packages Wiki Activity

Move sync (mutex) + cond + sigmask to OS layer

Browse source 
Migrates the mutex backend selection out of mutex.h/mutex.c, and the
background thread's condition-variable and signal-masking code out of
background_thread.c, onto the OS layer dispatcher pattern.
This commit is contained in:
guangli-dai 2026-07-27 17:08:58 -07:00 • committed by Guangli Dai
parent 67bca6a629
commit 192271fb82
14 changed files with 431 additions and 144 deletions
Download patch file Download diff file Expand all files Collapse all files

5
include/jemalloc/internal/background_thread.h
View file

@ -4,6 +4,7 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/base.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/os.h"
#if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
# define JEMALLOC_PTHREAD_CREATE_WRAPPER
@ -23,8 +24,8 @@ typedef enum {
struct background_thread_info_s {
#ifdef JEMALLOC_BACKGROUND_THREAD
/* Background thread is pthread specific. */
pthread_t thread;
pthread_cond_t cond;
pthread_t thread;
os_cond_t cond;
#endif
malloc_mutex_t mtx;
background_thread_state_t state;

73
include/jemalloc/internal/mutex.h
View file

@ -4,6 +4,7 @@
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/mutex_prof.h"
#include "jemalloc/internal/os.h"
#include "jemalloc/internal/tsd.h"
#include "jemalloc/internal/witness.h"
@ -40,19 +41,9 @@ struct malloc_mutex_s {
* before release), and may be read by other threads.
*/
atomic_b_t locked;
#ifdef _WIN32
# if _WIN32_WINNT >= 0x0600
SRWLOCK lock;
# else
CRITICAL_SECTION lock;
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
os_unfair_lock lock;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
pthread_mutex_t lock;
os_mutex_t lock;
#ifdef JEMALLOC_MUTEX_INIT_CB
malloc_mutex_t *postponed_next;
#else
pthread_mutex_t lock;
#endif
};
/*
@ -73,28 +64,9 @@ struct malloc_mutex_s {
#endif
};
#ifdef _WIN32
# if _WIN32_WINNT >= 0x0600
# define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
# define MALLOC_MUTEX_UNLOCK(m) \
ReleaseSRWLockExclusive(&(m)->lock)
# define MALLOC_MUTEX_TRYLOCK(m) \
(!TryAcquireSRWLockExclusive(&(m)->lock))
# else
# define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
# define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
# define MALLOC_MUTEX_TRYLOCK(m) \
(!TryEnterCriticalSection(&(m)->lock))
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
# define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
# define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock)
# define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
#else
# define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock)
# define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock)
# define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
#endif
#define MALLOC_MUTEX_LOCK(m) os_mutex_lock(&(m)->lock)
#define MALLOC_MUTEX_UNLOCK(m) os_mutex_unlock(&(m)->lock)
#define MALLOC_MUTEX_TRYLOCK(m) os_mutex_trylock(&(m)->lock)
#define LOCK_PROF_DATA_INITIALIZER \
{ \
@ -102,34 +74,15 @@ struct malloc_mutex_s {
ATOMIC_INIT(0), 0, NULL, 0 \
}
#ifdef _WIN32
#if !OS_MUTEX_HAS_STATIC_INIT
# define MALLOC_MUTEX_INITIALIZER
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
#elif defined(JEMALLOC_MUTEX_INIT_CB)
# if defined(JEMALLOC_DEBUG)
# define MALLOC_MUTEX_INITIALIZER \
{ \
{{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT), \
0 \
}
# else
# define MALLOC_MUTEX_INITIALIZER \
{ \
{{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT) \
}
# endif
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
# if (defined(JEMALLOC_DEBUG))
# define MALLOC_MUTEX_INITIALIZER \
{ \
{{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER, NULL}}, \
OS_MUTEX_INITIALIZER, NULL}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT), \
0 \
@ -139,20 +92,18 @@ struct malloc_mutex_s {
{ \
{{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER, NULL}}, \
OS_MUTEX_INITIALIZER, NULL}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT) \
}
# endif
#else
# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
# if defined(JEMALLOC_DEBUG)
# define MALLOC_MUTEX_INITIALIZER \
{ \
{{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER}}, \
OS_MUTEX_INITIALIZER}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT), \
0 \
@ -162,7 +113,7 @@ struct malloc_mutex_s {
{ \
{{LOCK_PROF_DATA_INITIALIZER, \
ATOMIC_INIT(false), \
PTHREAD_MUTEX_INITIALIZER}}, \
OS_MUTEX_INITIALIZER}}, \
WITNESS_INITIALIZER( \
"mutex", WITNESS_RANK_OMIT) \
}

9
include/jemalloc/internal/os.h
View file

@ -34,4 +34,13 @@
/* Time */
#include "jemalloc/internal/os/time.h"
/* Sync (mutex) */
#include "jemalloc/internal/os/mutex.h"
/* Cond (POSIX background thread only) */
#include "jemalloc/internal/os/cond.h"
/* Sigmask (POSIX background thread only) */
#include "jemalloc/internal/os/sigmask.h"
#endif /* JEMALLOC_INTERNAL_OS_H */

38
include/jemalloc/internal/os/cond.h Normal file
View file

@ -0,0 +1,38 @@
#ifndef JEMALLOC_INTERNAL_OS_COND_H
#define JEMALLOC_INTERNAL_OS_COND_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/os/detect.h"
#include "jemalloc/internal/os/mutex.h"
/*
* Condition-variable interface. Exists only when background threads are
* supported (JEMALLOC_BACKGROUND_THREAD, pthread-only, never defined on
* Windows, and also never defined on Darwin despite Darwin being POSIX, per
* configure.ac's os_unfair_lock/macho exclusion): this header is a no-op on
* any platform that doesn't need it, so os.h can include it unconditionally
* without forcing a backend to exist where none is required. Only once
* background threads ARE needed does the inner POSIX-vs-else split apply,
* enforcing a real backend (os/<os>/cond.h) for any such platform.
*
* As with os/mutex.h, the enforced function prototypes come AFTER the
* backend #include below: os_cond_t is a type (embedded by value in
* background_thread_info_t), not just a function parameter.
*/
#if defined(JEMALLOC_BACKGROUND_THREAD)
# if defined(JEMALLOC_OS_POSIX)
# include "jemalloc/internal/os/posix/cond.h"
# else
# error "OS layer: no cond backend for this platform; add os/<os>/cond.h"
# endif
/* Functions required for implementation in each backend. */
JEMALLOC_ALWAYS_INLINE bool os_cond_init(os_cond_t *c);
JEMALLOC_ALWAYS_INLINE int os_cond_wait(os_cond_t *c, os_mutex_t *m);
JEMALLOC_ALWAYS_INLINE int os_cond_timedwait(
os_cond_t *c, os_mutex_t *m, const struct timespec *abs_ts);
JEMALLOC_ALWAYS_INLINE void os_cond_signal(os_cond_t *c);
JEMALLOC_ALWAYS_INLINE void os_cond_now(struct timespec *now);
#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
#endif /* JEMALLOC_INTERNAL_OS_COND_H */

41
include/jemalloc/internal/os/darwin/mutex.h Normal file
View file

@ -0,0 +1,41 @@
#ifndef JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H
#define JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H
/*
* Darwin os_unfair_lock backend (macOS 10.12+, selected via
* JEMALLOC_OS_UNFAIR_LOCK at configure time). os_unfair_lock has no destroy
* and no postfork_child fixup beyond resetting to OS_UNFAIR_LOCK_INIT (it's
* a single atomic word). Older macOS without os_unfair_lock falls back to
* os/posix/mutex.h's pthread backend instead of this file (see the
* dispatcher in os/mutex.h).
*/
#include "jemalloc/internal/jemalloc_preamble.h"
#include <os/lock.h>
typedef os_unfair_lock os_mutex_t;
#define OS_MUTEX_INITIALIZER OS_UNFAIR_LOCK_INIT
#define OS_MUTEX_HAS_STATIC_INIT 1
JEMALLOC_ALWAYS_INLINE void
os_mutex_lock(os_mutex_t *m) {
os_unfair_lock_lock(m);
}
JEMALLOC_ALWAYS_INLINE void
os_mutex_unlock(os_mutex_t *m) {
os_unfair_lock_unlock(m);
}
JEMALLOC_ALWAYS_INLINE bool
os_mutex_trylock(os_mutex_t *m) {
return !os_unfair_lock_trylock(m);
}
JEMALLOC_ALWAYS_INLINE bool
os_mutex_init(os_mutex_t *m) {
*m = (os_unfair_lock)OS_UNFAIR_LOCK_INIT;
return false;
}
#endif /* JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H */

39
include/jemalloc/internal/os/mutex.h Normal file
View file

@ -0,0 +1,39 @@
#ifndef JEMALLOC_INTERNAL_OS_MUTEX_H
#define JEMALLOC_INTERNAL_OS_MUTEX_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/os/detect.h"
/*
* Sync (mutex) interface.
* Default: posix/. Override: Darwin (os_unfair_lock), Windows (SRWLOCK /
* CRITICAL_SECTION).
*
* Unlike the other os/<module>.h dispatchers, the enforced function
* prototypes come AFTER the backend #include below, not before: os_mutex_t
* is a type (embedded by value in malloc_mutex_t), not just a function
* parameter, so it must be fully defined by the backend before it can
* appear in a prototype at all.
*
* Capability macro: OS_MUTEX_HAS_STATIC_INIT: 1 iff OS_MUTEX_INITIALIZER is
* a valid static initializer for os_mutex_t, 0 if the mutex can only be
* initialized dynamically (via os_mutex_init()). Defined by whichever
* backend is selected below.
*/
#if defined(_WIN32)
# include "jemalloc/internal/os/windows/mutex.h"
#elif defined(JEMALLOC_OS_UNFAIR_LOCK)
# include "jemalloc/internal/os/darwin/mutex.h"
#elif defined(JEMALLOC_OS_POSIX)
# include "jemalloc/internal/os/posix/mutex.h"
#else
# error "OS layer: no mutex backend for this platform; add os/<os>/mutex.h"
#endif
/* Functions required for implementation in each backend. */
JEMALLOC_ALWAYS_INLINE void os_mutex_lock(os_mutex_t *m);
JEMALLOC_ALWAYS_INLINE void os_mutex_unlock(os_mutex_t *m);
JEMALLOC_ALWAYS_INLINE bool os_mutex_trylock(os_mutex_t *m);
JEMALLOC_ALWAYS_INLINE bool os_mutex_init(os_mutex_t *m);
#endif /* JEMALLOC_INTERNAL_OS_MUTEX_H */

75
include/jemalloc/internal/os/posix/cond.h Normal file
View file

@ -0,0 +1,75 @@
#ifndef JEMALLOC_INTERNAL_OS_POSIX_COND_H
#define JEMALLOC_INTERNAL_OS_POSIX_COND_H
/*
* POSIX pthread condvar backend. Solely backs the background thread.
*
* No os_cond_destroy: the background thread is the only cond user and never
* destroys its condvars (it never did, even pre-refactor), so it would be
* dead code. Add one here if a caller ever needs it.
*/
#include "jemalloc/internal/jemalloc_preamble.h"
typedef pthread_cond_t os_cond_t;
JEMALLOC_ALWAYS_INLINE bool
os_cond_init(os_cond_t *c) {
#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
/*
* Pair the condvar with CLOCK_MONOTONIC so os_cond_timedwait deadlines
* built from os_cond_now() are immune to wall-clock jumps.
*/
pthread_condattr_t attr;
if (pthread_condattr_init(&attr) != 0) {
return true;
}
if (pthread_condattr_setclock(&attr, CLOCK_MONOTONIC) != 0) {
pthread_condattr_destroy(&attr);
return true;
}
bool err = (pthread_cond_init(c, &attr) != 0);
pthread_condattr_destroy(&attr);
return err;
#else
return pthread_cond_init(c, NULL) != 0;
#endif
}
JEMALLOC_ALWAYS_INLINE int
os_cond_wait(os_cond_t *c, os_mutex_t *m) {
return pthread_cond_wait(c, m);
}
/*
* abs_ts is an absolute deadline in os_cond_now()'s clock (the clock the
* condvar was paired with in os_cond_init). Returns 0 on signal, ETIMEDOUT
* on timeout, other errno otherwise.
*/
JEMALLOC_ALWAYS_INLINE int
os_cond_timedwait(os_cond_t *c, os_mutex_t *m, const struct timespec *abs_ts) {
return pthread_cond_timedwait(c, m, abs_ts);
}
JEMALLOC_ALWAYS_INLINE void
os_cond_signal(os_cond_t *c) {
pthread_cond_signal(c);
}
/*
* Read "now" in the same clock os_cond_init() paired the condvar with, for
* building os_cond_timedwait() deadlines: CLOCK_MONOTONIC when available,
* else the wall clock.
*/
JEMALLOC_ALWAYS_INLINE void
os_cond_now(struct timespec *now) {
#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
clock_gettime(CLOCK_MONOTONIC, now);
#else
struct timeval tv;
gettimeofday(&tv, NULL);
now->tv_sec = tv.tv_sec;
now->tv_nsec = tv.tv_usec * 1000;
#endif
}
#endif /* JEMALLOC_INTERNAL_OS_POSIX_COND_H */

46
include/jemalloc/internal/os/posix/mutex.h Normal file
View file

@ -0,0 +1,46 @@
#ifndef JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H
#define JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H
/*
* POSIX pthread mutex backend. Default for every POSIX platform, including
* Darwin builds without os_unfair_lock (older macOS).
*/
#include "jemalloc/internal/jemalloc_preamble.h"
typedef pthread_mutex_t os_mutex_t;
#define OS_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
#define OS_MUTEX_HAS_STATIC_INIT 1
JEMALLOC_ALWAYS_INLINE void
os_mutex_lock(os_mutex_t *m) {
pthread_mutex_lock(m);
}
JEMALLOC_ALWAYS_INLINE void
os_mutex_unlock(os_mutex_t *m) {
pthread_mutex_unlock(m);
}
/* Returns true on failure (matches MALLOC_MUTEX_TRYLOCK semantics). */
JEMALLOC_ALWAYS_INLINE bool
os_mutex_trylock(os_mutex_t *m) {
return pthread_mutex_trylock(m) != 0;
}
/* Returns true on failure. */
JEMALLOC_ALWAYS_INLINE bool
os_mutex_init(os_mutex_t *m) {
pthread_mutexattr_t attr;
if (pthread_mutexattr_init(&attr) != 0) {
return true;
}
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
if (pthread_mutex_init(m, &attr) != 0) {
pthread_mutexattr_destroy(&attr);
return true;
}
pthread_mutexattr_destroy(&attr);
return false;
}
#endif /* JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H */

25
include/jemalloc/internal/os/posix/sigmask.h Normal file
View file

@ -0,0 +1,25 @@
#ifndef JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H
#define JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H
/*
* POSIX sigset_t-based signal-mask backend. Solely backs the background
* thread's pthread_create wrapper.
*/
#include "jemalloc/internal/jemalloc_preamble.h"
typedef sigset_t os_sigmask_t;
/* Mask all signals, returning the prior mask in *saved. */
JEMALLOC_ALWAYS_INLINE int
os_sigmask_all_enter(os_sigmask_t *saved) {
sigset_t set;
sigfillset(&set);
return pthread_sigmask(SIG_SETMASK, &set, saved);
}
JEMALLOC_ALWAYS_INLINE int
os_sigmask_leave(const os_sigmask_t *saved) {
return pthread_sigmask(SIG_SETMASK, saved, NULL);
}
#endif /* JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H */

29
include/jemalloc/internal/os/sigmask.h Normal file
View file

@ -0,0 +1,29 @@
#ifndef JEMALLOC_INTERNAL_OS_SIGMASK_H
#define JEMALLOC_INTERNAL_OS_SIGMASK_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/os/detect.h"
/*
* Signal-mask interface. Exists only when background threads are supported
* (JEMALLOC_BACKGROUND_THREAD), for the same reason as os/cond.h: it backs
* the background thread (masking signals during pthread_create so the new
* thread inherits an empty signal set), which is pthread-specific and never
* built on Windows -- and never enabled on Darwin either, despite Darwin
* being POSIX. Gating on JEMALLOC_BACKGROUND_THREAD first, rather than
* JEMALLOC_OS_POSIX alone, means this header is a no-op wherever background
* threads aren't needed, so os.h can include it unconditionally.
*/
#if defined(JEMALLOC_BACKGROUND_THREAD)
# if defined(JEMALLOC_OS_POSIX)
# include "jemalloc/internal/os/posix/sigmask.h"
# else
# error "OS layer: no sigmask backend for this platform; add os/<os>/sigmask.h"
# endif
/* Functions required for implementation in each backend. */
JEMALLOC_ALWAYS_INLINE int os_sigmask_all_enter(os_sigmask_t *saved);
JEMALLOC_ALWAYS_INLINE int os_sigmask_leave(const os_sigmask_t *saved);
#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
#endif /* JEMALLOC_INTERNAL_OS_SIGMASK_H */

72
include/jemalloc/internal/os/windows/mutex.h Normal file
View file

@ -0,0 +1,72 @@
#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H
#define JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H
/*
* Windows mutex backend. On Vista+, os_mutex_t is SRWLOCK (lighter than
* CRITICAL_SECTION); older targets fall back to CRITICAL_SECTION with a
* spin count.
*/
#include "jemalloc/internal/jemalloc_preamble.h"
#if _WIN32_WINNT >= 0x0600
typedef SRWLOCK os_mutex_t;
# define OS_MUTEX_INITIALIZER SRWLOCK_INIT
# define OS_MUTEX_HAS_STATIC_INIT 1
JEMALLOC_ALWAYS_INLINE void
os_mutex_lock(os_mutex_t *m) {
AcquireSRWLockExclusive(m);
}
JEMALLOC_ALWAYS_INLINE void
os_mutex_unlock(os_mutex_t *m) {
ReleaseSRWLockExclusive(m);
}
JEMALLOC_ALWAYS_INLINE bool
os_mutex_trylock(os_mutex_t *m) {
return !TryAcquireSRWLockExclusive(m);
}
JEMALLOC_ALWAYS_INLINE bool
os_mutex_init(os_mutex_t *m) {
InitializeSRWLock(m);
return false;
}
#else
typedef CRITICAL_SECTION os_mutex_t;
/*
* CRITICAL_SECTION has no static initializer, unlike SRWLOCK above -- so
* MALLOC_MUTEX_INITIALIZER is empty here, and jemalloc_init.c's init_lock
* (the one static malloc_mutex_t that must be usable before any other code
* runs, on every _WIN32_WINNT target) still needs its own constructor-based
* lazy-init workaround on this pre-Vista path.
*/
# define OS_MUTEX_HAS_STATIC_INIT 0
# ifndef _CRT_SPINCOUNT
# define _CRT_SPINCOUNT 4000
# endif
JEMALLOC_ALWAYS_INLINE void
os_mutex_lock(os_mutex_t *m) {
EnterCriticalSection(m);
}
JEMALLOC_ALWAYS_INLINE void
os_mutex_unlock(os_mutex_t *m) {
LeaveCriticalSection(m);
}
JEMALLOC_ALWAYS_INLINE bool
os_mutex_trylock(os_mutex_t *m) {
return !TryEnterCriticalSection(m);
}
JEMALLOC_ALWAYS_INLINE bool
os_mutex_init(os_mutex_t *m) {
return !InitializeCriticalSectionAndSpinCount(m, _CRT_SPINCOUNT);
}
#endif
#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H */

67
src/background_thread.c
View file

@ -10,6 +10,7 @@
#include "jemalloc/internal/jemalloc_internal_inlines_a.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/os.h"
#include "jemalloc/internal/prof.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/witness.h"
@ -77,7 +78,7 @@ background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind) {
assert(info->state == background_thread_paused);
info->state = background_thread_started;
#ifdef JEMALLOC_BACKGROUND_THREAD
pthread_cond_signal(&info->cond);
os_cond_signal(&info->cond);
#endif
malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
}
@ -247,59 +248,31 @@ background_thread_cond_wait(
int ret;
/*
* pthread_cond_wait drops and re-acquires the mutex internally, w/o
* going through our wrapper. Update the locked state explicitly.
* os_cond_wait drops and re-acquires the mutex internally, w/o going
* through our wrapper. Update the locked state explicitly.
*/
atomic_store_b(&info->mtx.locked, false, ATOMIC_RELAXED);
if (ts == NULL) {
ret = pthread_cond_wait(&info->cond, &info->mtx.lock);
ret = os_cond_wait(&info->cond, &info->mtx.lock);
} else {
ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, ts);
ret = os_cond_timedwait(&info->cond, &info->mtx.lock, ts);
}
atomic_store_b(&info->mtx.locked, true, ATOMIC_RELAXED);
return ret;
}
static int
background_thread_cond_init(pthread_cond_t *cond) {
#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
pthread_condattr_t cond_attr;
int ret = pthread_condattr_init(&cond_attr);
if (ret != 0) {
return ret;
}
ret = pthread_condattr_setclock(&cond_attr, CLOCK_MONOTONIC);
if (ret != 0) {
pthread_condattr_destroy(&cond_attr);
return ret;
}
ret = pthread_cond_init(cond, &cond_attr);
pthread_condattr_destroy(&cond_attr);
return ret;
#else
return pthread_cond_init(cond, NULL);
#endif
}
/*
* Fill in the absolute deadline for pthread_cond_timedwait. The clock read
* here MUST match the clock the condvar was initialized with in
* background_thread_cond_init, otherwise the deadline is interpreted against
* the wrong epoch.
* Fill in the absolute deadline for os_cond_timedwait. os_cond_now() reads
* the same clock os_cond_init() paired the condvar with, so the deadline
* lands in the right epoch.
*/
static void
background_thread_wakeup_ts_init(struct timespec *ts, uint64_t interval) {
nstime_t wakeup;
#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
os_cond_now(&now);
nstime_init2(&wakeup, now.tv_sec, now.tv_nsec);
#else
struct timeval tv;
gettimeofday(&tv, NULL);
nstime_init2(&wakeup, tv.tv_sec, tv.tv_usec * 1000);
#endif
nstime_iadd(&wakeup, interval);
ts->tv_sec = (size_t)nstime_sec(&wakeup);
ts->tv_nsec = (size_t)nstime_nsec(&wakeup);
@ -426,7 +399,7 @@ background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) {
if (info->state == background_thread_started) {
has_thread = true;
info->state = background_thread_stopped;
pthread_cond_signal(&info->cond);
os_cond_signal(&info->cond);
} else {
has_thread = false;
}
@ -457,10 +430,8 @@ background_thread_create_signals_masked(pthread_t *thread,
* Mask signals during thread creation so that the thread inherits
* an empty signal set.
*/
sigset_t set;
sigfillset(&set);
sigset_t oldset;
int mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset);
os_sigmask_t oldset;
int mask_err = os_sigmask_all_enter(&oldset);
if (mask_err != 0) {
return mask_err;
}
@ -470,7 +441,7 @@ background_thread_create_signals_masked(pthread_t *thread,
* Restore the signal mask. Failure to restore the signal mask here
* changes program behavior.
*/
int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
int restore_err = os_sigmask_leave(&oldset);
if (restore_err != 0) {
malloc_printf(
"<jemalloc>: background thread creation "
@ -681,7 +652,7 @@ background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) {
/* Threads are created asynchronously by Thread 0. */
background_thread_info_t *t0 = &background_thread_info[0];
malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx);
pthread_cond_signal(&t0->cond);
os_cond_signal(&t0->cond);
malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx);
return false;
@ -813,7 +784,7 @@ background_thread_wakeup_early(
&& nstime_ns(remaining_sleep) < BACKGROUND_THREAD_MIN_INTERVAL_NS) {
return;
}
pthread_cond_signal(&info->cond);
os_cond_signal(&info->cond);
}
void
@ -857,8 +828,8 @@ background_thread_postfork_child(tsdn_t *tsdn) {
background_thread_info_t *info = &background_thread_info[i];
malloc_mutex_lock(tsdn, &info->mtx);
info->state = background_thread_stopped;
int ret = background_thread_cond_init(&info->cond);
assert(ret == 0);
bool ret = os_cond_init(&info->cond);
assert(!ret);
background_thread_info_init(tsdn, info);
malloc_mutex_unlock(tsdn, &info->mtx);
}
@ -975,7 +946,7 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) {
malloc_mutex_address_ordered)) {
return true;
}
if (background_thread_cond_init(&info->cond)) {
if (os_cond_init(&info->cond)) {
return true;
}
malloc_mutex_lock(tsdn, &info->mtx);

19
src/jemalloc_init.c
View file

@ -55,10 +55,14 @@ malloc_initializer_set(void) {
}
/* Used to avoid initialization races. */
#ifdef _WIN32
# if _WIN32_WINNT >= 0x0600
static malloc_mutex_t init_lock = SRWLOCK_INIT;
# else
#if !OS_MUTEX_HAS_STATIC_INIT
/*
* This target's os_mutex_t has no static initializer (currently: pre-Vista
* Windows CRITICAL_SECTION; see os/windows/mutex.h), so MALLOC_MUTEX_
* INITIALIZER is empty -- init_lock needs its own lazy-init workaround
* instead, being the one static malloc_mutex_t that must be usable before
* any other code runs.
*/
static malloc_mutex_t init_lock;
static bool init_lock_initialized = false;
@ -82,12 +86,11 @@ _init_init_lock(void) {
init_lock_initialized = true;
}
# ifdef _MSC_VER
# pragma section(".CRT$XCU", read)
# ifdef _MSC_VER
# pragma section(".CRT$XCU", read)
JEMALLOC_SECTION(".CRT$XCU")
JEMALLOC_ATTR(used)
static const void(WINAPI *init_init_lock)(void) = _init_init_lock;
# endif
# endif
#else
static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
@ -599,7 +602,7 @@ malloc_init_hard(void) {
*/
assert(SC_NTINY == 0 || SC_LG_TINY_MAXCLASS <= SC_LG_LARGE_MINCLASS);
#if defined(_WIN32) && _WIN32_WINNT < 0x0600
#if !OS_MUTEX_HAS_STATIC_INIT
_init_init_lock();
#endif
malloc_mutex_lock(TSDN_NULL, &init_lock);

37
src/mutex.c
View file

@ -5,10 +5,6 @@
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/spin.h"
#if defined(_WIN32) && !defined(_CRT_SPINCOUNT)
# define _CRT_SPINCOUNT 4000
#endif
/*
* Based on benchmark results, a fixed spin with this amount of retries works
* well for our critical sections.
@ -141,18 +137,17 @@ bool
malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank,
malloc_mutex_lock_order_t lock_order) {
mutex_prof_data_init(&mutex->prof_data);
#ifdef _WIN32
# if _WIN32_WINNT >= 0x0600
InitializeSRWLock(&mutex->lock);
# else
if (!InitializeCriticalSectionAndSpinCount(
&mutex->lock, _CRT_SPINCOUNT)) {
return true;
}
# endif
#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
mutex->lock = OS_UNFAIR_LOCK_INIT;
#elif (defined(JEMALLOC_MUTEX_INIT_CB))
#ifdef JEMALLOC_MUTEX_INIT_CB
/*
* Targets platforms with _pthread_mutex_init_calloc_cb(), which is
* currently just FreeBSD/libthr: its pthread_mutex_init() can calloc()
* internally, which would recurse back into jemalloc before it is safe
* safe to allocate. Until malloc_mutex_boot() runs, defer real
* initialization by queuing the mutex here instead; malloc_mutex_boot()
* later drains the queue, calling _pthread_mutex_init_calloc_cb() with
* bootstrap_calloc (jemalloc's own early-bootstrap allocation path) as
* the callback, once that's safe to use.
*/
if (postpone_init) {
mutex->postponed_next = postponed_mutexes;
postponed_mutexes = mutex;
@ -164,17 +159,9 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank,
}
}
#else
pthread_mutexattr_t attr;
if (pthread_mutexattr_init(&attr) != 0) {
if (os_mutex_init(&mutex->lock)) {
return true;
}
pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
if (pthread_mutex_init(&mutex->lock, &attr) != 0) {
pthread_mutexattr_destroy(&attr);
return true;
}
pthread_mutexattr_destroy(&attr);
#endif
if (config_debug) {
mutex->lock_order = lock_order;