Move sync (mutex) + cond + sigmask to OS layer

Migrates the mutex backend selection out of mutex.h/mutex.c, and the
background thread's condition-variable and signal-masking code out of
background_thread.c, onto the OS layer dispatcher pattern.

include/jemalloc/internal/background_thread.h

@@ -4,6 +4,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/base.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/os.h"
 
 #if defined(JEMALLOC_BACKGROUND_THREAD) || defined(JEMALLOC_LAZY_LOCK)
 #	define JEMALLOC_PTHREAD_CREATE_WRAPPER
@@ -23,8 +24,8 @@ typedef enum {
 struct background_thread_info_s {
 #ifdef JEMALLOC_BACKGROUND_THREAD
 	/* Background thread is pthread specific. */
-	pthread_t      thread;
+	pthread_t thread;
-	pthread_cond_t cond;
+	os_cond_t cond;
 #endif
 	malloc_mutex_t            mtx;
 	background_thread_state_t state;

include/jemalloc/internal/mutex.h

@@ -4,6 +4,7 @@
 #include "jemalloc/internal/jemalloc_preamble.h"
 #include "jemalloc/internal/atomic.h"
 #include "jemalloc/internal/mutex_prof.h"
+#include "jemalloc/internal/os.h"
 #include "jemalloc/internal/tsd.h"
 #include "jemalloc/internal/witness.h"
 
@@ -40,19 +41,9 @@ struct malloc_mutex_s {
 			 * before release), and may be read by other threads.
 			 */
 			atomic_b_t locked;
-#ifdef _WIN32
+			os_mutex_t lock;
-#	if _WIN32_WINNT >= 0x0600
+#ifdef JEMALLOC_MUTEX_INIT_CB
-			SRWLOCK lock;
-#	else
-			CRITICAL_SECTION lock;
-#	endif
-#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
-			os_unfair_lock lock;
-#elif (defined(JEMALLOC_MUTEX_INIT_CB))
-			pthread_mutex_t lock;
 			malloc_mutex_t *postponed_next;
-#else
-			pthread_mutex_t lock;
 #endif
 		};
 		/*
@@ -73,28 +64,9 @@ struct malloc_mutex_s {
 #endif
 };
 
-#ifdef _WIN32
+#define MALLOC_MUTEX_LOCK(m) os_mutex_lock(&(m)->lock)
-#	if _WIN32_WINNT >= 0x0600
+#define MALLOC_MUTEX_UNLOCK(m) os_mutex_unlock(&(m)->lock)
-#		define MALLOC_MUTEX_LOCK(m) AcquireSRWLockExclusive(&(m)->lock)
+#define MALLOC_MUTEX_TRYLOCK(m) os_mutex_trylock(&(m)->lock)
-#		define MALLOC_MUTEX_UNLOCK(m)                                 \
-			ReleaseSRWLockExclusive(&(m)->lock)
-#		define MALLOC_MUTEX_TRYLOCK(m)                                \
-			(!TryAcquireSRWLockExclusive(&(m)->lock))
-#	else
-#		define MALLOC_MUTEX_LOCK(m) EnterCriticalSection(&(m)->lock)
-#		define MALLOC_MUTEX_UNLOCK(m) LeaveCriticalSection(&(m)->lock)
-#		define MALLOC_MUTEX_TRYLOCK(m)                                \
-			(!TryEnterCriticalSection(&(m)->lock))
-#	endif
-#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
-#	define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock)
-#	define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock)
-#	define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
-#else
-#	define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock)
-#	define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock)
-#	define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
-#endif
 
 #define LOCK_PROF_DATA_INITIALIZER                                             \
 	{                                                                      \
@@ -102,34 +74,15 @@ struct malloc_mutex_s {
 		    ATOMIC_INIT(0), 0, NULL, 0                                 \
 	}
 
-#ifdef _WIN32
+#if !OS_MUTEX_HAS_STATIC_INIT
 #	define MALLOC_MUTEX_INITIALIZER
-#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
+#elif defined(JEMALLOC_MUTEX_INIT_CB)
 #	if defined(JEMALLOC_DEBUG)
-#		define MALLOC_MUTEX_INITIALIZER                               \
-			{                                                      \
-				{{LOCK_PROF_DATA_INITIALIZER,                  \
-				    ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
-				    WITNESS_INITIALIZER(                       \
-				        "mutex", WITNESS_RANK_OMIT),           \
-				    0                                          \
-			}
-#	else
-#		define MALLOC_MUTEX_INITIALIZER                               \
-			{                                                      \
-				{{LOCK_PROF_DATA_INITIALIZER,                  \
-				    ATOMIC_INIT(false), OS_UNFAIR_LOCK_INIT}}, \
-				    WITNESS_INITIALIZER(                       \
-				        "mutex", WITNESS_RANK_OMIT)            \
-			}
-#	endif
-#elif (defined(JEMALLOC_MUTEX_INIT_CB))
-#	if (defined(JEMALLOC_DEBUG))
 #		define MALLOC_MUTEX_INITIALIZER                               \
 			{                                                      \
 				{{LOCK_PROF_DATA_INITIALIZER,                  \
 				    ATOMIC_INIT(false),                        \
-				    PTHREAD_MUTEX_INITIALIZER, NULL}},         \
+				    OS_MUTEX_INITIALIZER, NULL}},              \
 				    WITNESS_INITIALIZER(                       \
 				        "mutex", WITNESS_RANK_OMIT),           \
 				    0                                          \
@@ -139,20 +92,18 @@ struct malloc_mutex_s {
 			{                                                      \
 				{{LOCK_PROF_DATA_INITIALIZER,                  \
 				    ATOMIC_INIT(false),                        \
-				    PTHREAD_MUTEX_INITIALIZER, NULL}},         \
+				    OS_MUTEX_INITIALIZER, NULL}},              \
 				    WITNESS_INITIALIZER(                       \
 				        "mutex", WITNESS_RANK_OMIT)            \
 			}
 #	endif
-
 #else
-#	define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
 #	if defined(JEMALLOC_DEBUG)
 #		define MALLOC_MUTEX_INITIALIZER                               \
 			{                                                      \
 				{{LOCK_PROF_DATA_INITIALIZER,                  \
 				    ATOMIC_INIT(false),                        \
-				    PTHREAD_MUTEX_INITIALIZER}},               \
+				    OS_MUTEX_INITIALIZER}},                    \
 				    WITNESS_INITIALIZER(                       \
 				        "mutex", WITNESS_RANK_OMIT),           \
 				    0                                          \
@@ -162,7 +113,7 @@ struct malloc_mutex_s {
 			{                                                      \
 				{{LOCK_PROF_DATA_INITIALIZER,                  \
 				    ATOMIC_INIT(false),                        \
-				    PTHREAD_MUTEX_INITIALIZER}},               \
+				    OS_MUTEX_INITIALIZER}},                    \
 				    WITNESS_INITIALIZER(                       \
 				        "mutex", WITNESS_RANK_OMIT)            \
 			}

include/jemalloc/internal/os.h

@@ -34,4 +34,13 @@
 /* Time */
 #include "jemalloc/internal/os/time.h"
 
+/* Sync (mutex) */
+#include "jemalloc/internal/os/mutex.h"
+
+/* Cond (POSIX background thread only) */
+#include "jemalloc/internal/os/cond.h"
+
+/* Sigmask (POSIX background thread only) */
+#include "jemalloc/internal/os/sigmask.h"
+
 #endif /* JEMALLOC_INTERNAL_OS_H */

include/jemalloc/internal/os/cond.h

@@ -0,0 +1,38 @@
+#ifndef JEMALLOC_INTERNAL_OS_COND_H
+#define JEMALLOC_INTERNAL_OS_COND_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/os/detect.h"
+#include "jemalloc/internal/os/mutex.h"
+
+/*
+ * Condition-variable interface.  Exists only when background threads are
+ * supported (JEMALLOC_BACKGROUND_THREAD, pthread-only, never defined on
+ * Windows, and also never defined on Darwin despite Darwin being POSIX, per
+ * configure.ac's os_unfair_lock/macho exclusion): this header is a no-op on
+ * any platform that doesn't need it, so os.h can include it unconditionally
+ * without forcing a backend to exist where none is required.  Only once
+ * background threads ARE needed does the inner POSIX-vs-else split apply,
+ * enforcing a real backend (os/<os>/cond.h) for any such platform.
+ *
+ * As with os/mutex.h, the enforced function prototypes come AFTER the
+ * backend #include below: os_cond_t is a type (embedded by value in
+ * background_thread_info_t), not just a function parameter.
+ */
+#if defined(JEMALLOC_BACKGROUND_THREAD)
+#  if defined(JEMALLOC_OS_POSIX)
+#    include "jemalloc/internal/os/posix/cond.h"
+#  else
+#    error "OS layer: no cond backend for this platform; add os/<os>/cond.h"
+#  endif
+
+/* Functions required for implementation in each backend. */
+JEMALLOC_ALWAYS_INLINE bool os_cond_init(os_cond_t *c);
+JEMALLOC_ALWAYS_INLINE int os_cond_wait(os_cond_t *c, os_mutex_t *m);
+JEMALLOC_ALWAYS_INLINE int os_cond_timedwait(
+    os_cond_t *c, os_mutex_t *m, const struct timespec *abs_ts);
+JEMALLOC_ALWAYS_INLINE void os_cond_signal(os_cond_t *c);
+JEMALLOC_ALWAYS_INLINE void os_cond_now(struct timespec *now);
+#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
+
+#endif /* JEMALLOC_INTERNAL_OS_COND_H */

include/jemalloc/internal/os/darwin/mutex.h

@@ -0,0 +1,41 @@
+#ifndef JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H
+#define JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H
+
+/*
+ * Darwin os_unfair_lock backend (macOS 10.12+, selected via
+ * JEMALLOC_OS_UNFAIR_LOCK at configure time). os_unfair_lock has no destroy
+ * and no postfork_child fixup beyond resetting to OS_UNFAIR_LOCK_INIT (it's
+ * a single atomic word).  Older macOS without os_unfair_lock falls back to
+ * os/posix/mutex.h's pthread backend instead of this file (see the
+ * dispatcher in os/mutex.h).
+ */
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+#include <os/lock.h>
+
+typedef os_unfair_lock os_mutex_t;
+#define OS_MUTEX_INITIALIZER OS_UNFAIR_LOCK_INIT
+#define OS_MUTEX_HAS_STATIC_INIT 1
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_lock(os_mutex_t *m) {
+	os_unfair_lock_lock(m);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_unlock(os_mutex_t *m) {
+	os_unfair_lock_unlock(m);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_trylock(os_mutex_t *m) {
+	return !os_unfair_lock_trylock(m);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_init(os_mutex_t *m) {
+	*m = (os_unfair_lock)OS_UNFAIR_LOCK_INIT;
+	return false;
+}
+
+#endif /* JEMALLOC_INTERNAL_OS_DARWIN_MUTEX_H */

include/jemalloc/internal/os/mutex.h

@@ -0,0 +1,39 @@
+#ifndef JEMALLOC_INTERNAL_OS_MUTEX_H
+#define JEMALLOC_INTERNAL_OS_MUTEX_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/os/detect.h"
+
+/*
+ * Sync (mutex) interface.
+ * Default: posix/.  Override: Darwin (os_unfair_lock), Windows (SRWLOCK /
+ * CRITICAL_SECTION).
+ *
+ * Unlike the other os/<module>.h dispatchers, the enforced function
+ * prototypes come AFTER the backend #include below, not before: os_mutex_t
+ * is a type (embedded by value in malloc_mutex_t), not just a function
+ * parameter, so it must be fully defined by the backend before it can
+ * appear in a prototype at all.
+ *
+ * Capability macro: OS_MUTEX_HAS_STATIC_INIT: 1 iff OS_MUTEX_INITIALIZER is
+ * a valid static initializer for os_mutex_t, 0 if the mutex can only be
+ * initialized dynamically (via os_mutex_init()).  Defined by whichever
+ * backend is selected below.
+ */
+#if defined(_WIN32)
+#  include "jemalloc/internal/os/windows/mutex.h"
+#elif defined(JEMALLOC_OS_UNFAIR_LOCK)
+#  include "jemalloc/internal/os/darwin/mutex.h"
+#elif defined(JEMALLOC_OS_POSIX)
+#  include "jemalloc/internal/os/posix/mutex.h"
+#else
+#  error "OS layer: no mutex backend for this platform; add os/<os>/mutex.h"
+#endif
+
+/* Functions required for implementation in each backend. */
+JEMALLOC_ALWAYS_INLINE void os_mutex_lock(os_mutex_t *m);
+JEMALLOC_ALWAYS_INLINE void os_mutex_unlock(os_mutex_t *m);
+JEMALLOC_ALWAYS_INLINE bool os_mutex_trylock(os_mutex_t *m);
+JEMALLOC_ALWAYS_INLINE bool os_mutex_init(os_mutex_t *m);
+
+#endif /* JEMALLOC_INTERNAL_OS_MUTEX_H */

include/jemalloc/internal/os/posix/cond.h

@@ -0,0 +1,75 @@
+#ifndef JEMALLOC_INTERNAL_OS_POSIX_COND_H
+#define JEMALLOC_INTERNAL_OS_POSIX_COND_H
+
+/*
+ * POSIX pthread condvar backend.  Solely backs the background thread.
+ *
+ * No os_cond_destroy: the background thread is the only cond user and never
+ * destroys its condvars (it never did, even pre-refactor), so it would be
+ * dead code.  Add one here if a caller ever needs it.
+ */
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+typedef pthread_cond_t os_cond_t;
+
+JEMALLOC_ALWAYS_INLINE bool
+os_cond_init(os_cond_t *c) {
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	/*
+	 * Pair the condvar with CLOCK_MONOTONIC so os_cond_timedwait deadlines
+	 * built from os_cond_now() are immune to wall-clock jumps.
+	 */
+	pthread_condattr_t attr;
+	if (pthread_condattr_init(&attr) != 0) {
+		return true;
+	}
+	if (pthread_condattr_setclock(&attr, CLOCK_MONOTONIC) != 0) {
+		pthread_condattr_destroy(&attr);
+		return true;
+	}
+	bool err = (pthread_cond_init(c, &attr) != 0);
+	pthread_condattr_destroy(&attr);
+	return err;
+#else
+	return pthread_cond_init(c, NULL) != 0;
+#endif
+}
+
+JEMALLOC_ALWAYS_INLINE int
+os_cond_wait(os_cond_t *c, os_mutex_t *m) {
+	return pthread_cond_wait(c, m);
+}
+
+/*
+ * abs_ts is an absolute deadline in os_cond_now()'s clock (the clock the
+ * condvar was paired with in os_cond_init).  Returns 0 on signal, ETIMEDOUT
+ * on timeout, other errno otherwise.
+ */
+JEMALLOC_ALWAYS_INLINE int
+os_cond_timedwait(os_cond_t *c, os_mutex_t *m, const struct timespec *abs_ts) {
+	return pthread_cond_timedwait(c, m, abs_ts);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_cond_signal(os_cond_t *c) {
+	pthread_cond_signal(c);
+}
+
+/*
+ * Read "now" in the same clock os_cond_init() paired the condvar with, for
+ * building os_cond_timedwait() deadlines: CLOCK_MONOTONIC when available,
+ * else the wall clock.
+ */
+JEMALLOC_ALWAYS_INLINE void
+os_cond_now(struct timespec *now) {
+#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
+	clock_gettime(CLOCK_MONOTONIC, now);
+#else
+	struct timeval tv;
+	gettimeofday(&tv, NULL);
+	now->tv_sec = tv.tv_sec;
+	now->tv_nsec = tv.tv_usec * 1000;
+#endif
+}
+
+#endif /* JEMALLOC_INTERNAL_OS_POSIX_COND_H */

include/jemalloc/internal/os/posix/mutex.h

@@ -0,0 +1,46 @@
+#ifndef JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H
+#define JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H
+
+/*
+ * POSIX pthread mutex backend.  Default for every POSIX platform, including
+ * Darwin builds without os_unfair_lock (older macOS).
+ */
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+typedef pthread_mutex_t os_mutex_t;
+#define OS_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#define OS_MUTEX_HAS_STATIC_INIT 1
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_lock(os_mutex_t *m) {
+	pthread_mutex_lock(m);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_unlock(os_mutex_t *m) {
+	pthread_mutex_unlock(m);
+}
+
+/* Returns true on failure (matches MALLOC_MUTEX_TRYLOCK semantics). */
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_trylock(os_mutex_t *m) {
+	return pthread_mutex_trylock(m) != 0;
+}
+
+/* Returns true on failure. */
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_init(os_mutex_t *m) {
+	pthread_mutexattr_t attr;
+	if (pthread_mutexattr_init(&attr) != 0) {
+		return true;
+	}
+	pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT);
+	if (pthread_mutex_init(m, &attr) != 0) {
+		pthread_mutexattr_destroy(&attr);
+		return true;
+	}
+	pthread_mutexattr_destroy(&attr);
+	return false;
+}
+
+#endif /* JEMALLOC_INTERNAL_OS_POSIX_MUTEX_H */

include/jemalloc/internal/os/posix/sigmask.h

@@ -0,0 +1,25 @@
+#ifndef JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H
+#define JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H
+
+/*
+ * POSIX sigset_t-based signal-mask backend.  Solely backs the background
+ * thread's pthread_create wrapper.
+ */
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+typedef sigset_t os_sigmask_t;
+
+/* Mask all signals, returning the prior mask in *saved. */
+JEMALLOC_ALWAYS_INLINE int
+os_sigmask_all_enter(os_sigmask_t *saved) {
+	sigset_t set;
+	sigfillset(&set);
+	return pthread_sigmask(SIG_SETMASK, &set, saved);
+}
+
+JEMALLOC_ALWAYS_INLINE int
+os_sigmask_leave(const os_sigmask_t *saved) {
+	return pthread_sigmask(SIG_SETMASK, saved, NULL);
+}
+
+#endif /* JEMALLOC_INTERNAL_OS_POSIX_SIGMASK_H */

include/jemalloc/internal/os/sigmask.h

@@ -0,0 +1,29 @@
+#ifndef JEMALLOC_INTERNAL_OS_SIGMASK_H
+#define JEMALLOC_INTERNAL_OS_SIGMASK_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/os/detect.h"
+
+/*
+ * Signal-mask interface.  Exists only when background threads are supported
+ * (JEMALLOC_BACKGROUND_THREAD), for the same reason as os/cond.h: it backs
+ * the background thread (masking signals during pthread_create so the new
+ * thread inherits an empty signal set), which is pthread-specific and never
+ * built on Windows -- and never enabled on Darwin either, despite Darwin
+ * being POSIX.  Gating on JEMALLOC_BACKGROUND_THREAD first, rather than
+ * JEMALLOC_OS_POSIX alone, means this header is a no-op wherever background
+ * threads aren't needed, so os.h can include it unconditionally.
+ */
+#if defined(JEMALLOC_BACKGROUND_THREAD)
+#  if defined(JEMALLOC_OS_POSIX)
+#    include "jemalloc/internal/os/posix/sigmask.h"
+#  else
+#    error "OS layer: no sigmask backend for this platform; add os/<os>/sigmask.h"
+#  endif
+
+/* Functions required for implementation in each backend. */
+JEMALLOC_ALWAYS_INLINE int os_sigmask_all_enter(os_sigmask_t *saved);
+JEMALLOC_ALWAYS_INLINE int os_sigmask_leave(const os_sigmask_t *saved);
+#endif /* defined(JEMALLOC_BACKGROUND_THREAD) */
+
+#endif /* JEMALLOC_INTERNAL_OS_SIGMASK_H */

include/jemalloc/internal/os/windows/mutex.h

@@ -0,0 +1,72 @@
+#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H
+#define JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H
+
+/*
+ * Windows mutex backend.  On Vista+, os_mutex_t is SRWLOCK (lighter than
+ * CRITICAL_SECTION); older targets fall back to CRITICAL_SECTION with a
+ * spin count.
+ */
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+#if _WIN32_WINNT >= 0x0600
+typedef SRWLOCK os_mutex_t;
+#  define OS_MUTEX_INITIALIZER SRWLOCK_INIT
+#  define OS_MUTEX_HAS_STATIC_INIT 1
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_lock(os_mutex_t *m) {
+	AcquireSRWLockExclusive(m);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_unlock(os_mutex_t *m) {
+	ReleaseSRWLockExclusive(m);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_trylock(os_mutex_t *m) {
+	return !TryAcquireSRWLockExclusive(m);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_init(os_mutex_t *m) {
+	InitializeSRWLock(m);
+	return false;
+}
+#else
+typedef CRITICAL_SECTION os_mutex_t;
+/*
+ * CRITICAL_SECTION has no static initializer, unlike SRWLOCK above -- so
+ * MALLOC_MUTEX_INITIALIZER is empty here, and jemalloc_init.c's init_lock
+ * (the one static malloc_mutex_t that must be usable before any other code
+ * runs, on every _WIN32_WINNT target) still needs its own constructor-based
+ * lazy-init workaround on this pre-Vista path.
+ */
+#  define OS_MUTEX_HAS_STATIC_INIT 0
+
+#  ifndef _CRT_SPINCOUNT
+#    define _CRT_SPINCOUNT 4000
+#  endif
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_lock(os_mutex_t *m) {
+	EnterCriticalSection(m);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_mutex_unlock(os_mutex_t *m) {
+	LeaveCriticalSection(m);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_trylock(os_mutex_t *m) {
+	return !TryEnterCriticalSection(m);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_mutex_init(os_mutex_t *m) {
+	return !InitializeCriticalSectionAndSpinCount(m, _CRT_SPINCOUNT);
+}
+#endif
+
+#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_MUTEX_H */

src/background_thread.c

@@ -10,6 +10,7 @@
 #include "jemalloc/internal/jemalloc_internal_inlines_a.h"
 #include "jemalloc/internal/malloc_io.h"
 #include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/os.h"
 #include "jemalloc/internal/prof.h"
 #include "jemalloc/internal/tcache.h"
 #include "jemalloc/internal/witness.h"
@@ -77,7 +78,7 @@ background_thread_arena_reset_finish(tsd_t *tsd, unsigned arena_ind) {
 			assert(info->state == background_thread_paused);
 			info->state = background_thread_started;
 #ifdef JEMALLOC_BACKGROUND_THREAD
-			pthread_cond_signal(&info->cond);
+			os_cond_signal(&info->cond);
 #endif
 			malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx);
 		}
@@ -247,59 +248,31 @@ background_thread_cond_wait(
 	int ret;
 
 	/*
-	 * pthread_cond_wait drops and re-acquires the mutex internally, w/o
+	 * os_cond_wait drops and re-acquires the mutex internally, w/o going
-	 * going through our wrapper.  Update the locked state explicitly.
+	 * through our wrapper.  Update the locked state explicitly.
 	 */
 	atomic_store_b(&info->mtx.locked, false, ATOMIC_RELAXED);
 	if (ts == NULL) {
-		ret = pthread_cond_wait(&info->cond, &info->mtx.lock);
+		ret = os_cond_wait(&info->cond, &info->mtx.lock);
 	} else {
-		ret = pthread_cond_timedwait(&info->cond, &info->mtx.lock, ts);
+		ret = os_cond_timedwait(&info->cond, &info->mtx.lock, ts);
 	}
 	atomic_store_b(&info->mtx.locked, true, ATOMIC_RELAXED);
 
 	return ret;
 }
 
-static int
-background_thread_cond_init(pthread_cond_t *cond) {
-#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
-	pthread_condattr_t cond_attr;
-	int ret = pthread_condattr_init(&cond_attr);
-	if (ret != 0) {
-		return ret;
-	}
-	ret = pthread_condattr_setclock(&cond_attr, CLOCK_MONOTONIC);
-	if (ret != 0) {
-		pthread_condattr_destroy(&cond_attr);
-		return ret;
-	}
-	ret = pthread_cond_init(cond, &cond_attr);
-	pthread_condattr_destroy(&cond_attr);
-	return ret;
-#else
-	return pthread_cond_init(cond, NULL);
-#endif
-}
-
 /*
- * Fill in the absolute deadline for pthread_cond_timedwait.  The clock read
+ * Fill in the absolute deadline for os_cond_timedwait.  os_cond_now() reads
- * here MUST match the clock the condvar was initialized with in
+ * the same clock os_cond_init() paired the condvar with, so the deadline
- * background_thread_cond_init, otherwise the deadline is interpreted against
+ * lands in the right epoch.
- * the wrong epoch.
  */
 static void
 background_thread_wakeup_ts_init(struct timespec *ts, uint64_t interval) {
 	nstime_t wakeup;
-#ifdef JEMALLOC_HAVE_PTHREAD_COND_TIMEDWAIT_MONOTONIC
 	struct timespec now;
-	clock_gettime(CLOCK_MONOTONIC, &now);
+	os_cond_now(&now);
 	nstime_init2(&wakeup, now.tv_sec, now.tv_nsec);
-#else
-	struct timeval tv;
-	gettimeofday(&tv, NULL);
-	nstime_init2(&wakeup, tv.tv_sec, tv.tv_usec * 1000);
-#endif
 	nstime_iadd(&wakeup, interval);
 	ts->tv_sec = (size_t)nstime_sec(&wakeup);
 	ts->tv_nsec = (size_t)nstime_nsec(&wakeup);
@@ -426,7 +399,7 @@ background_threads_disable_single(tsd_t *tsd, background_thread_info_t *info) {
 	if (info->state == background_thread_started) {
 		has_thread = true;
 		info->state = background_thread_stopped;
-		pthread_cond_signal(&info->cond);
+		os_cond_signal(&info->cond);
 	} else {
 		has_thread = false;
 	}
@@ -457,10 +430,8 @@ background_thread_create_signals_masked(pthread_t *thread,
 	 * Mask signals during thread creation so that the thread inherits
 	 * an empty signal set.
 	 */
-	sigset_t set;
+	os_sigmask_t oldset;
-	sigfillset(&set);
+	int          mask_err = os_sigmask_all_enter(&oldset);
-	sigset_t oldset;
-	int      mask_err = pthread_sigmask(SIG_SETMASK, &set, &oldset);
 	if (mask_err != 0) {
 		return mask_err;
 	}
@@ -470,7 +441,7 @@ background_thread_create_signals_masked(pthread_t *thread,
 	 * Restore the signal mask.  Failure to restore the signal mask here
 	 * changes program behavior.
 	 */
-	int restore_err = pthread_sigmask(SIG_SETMASK, &oldset, NULL);
+	int restore_err = os_sigmask_leave(&oldset);
 	if (restore_err != 0) {
 		malloc_printf(
 		    "<jemalloc>: background thread creation "
@@ -681,7 +652,7 @@ background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) {
 		/* Threads are created asynchronously by Thread 0. */
 		background_thread_info_t *t0 = &background_thread_info[0];
 		malloc_mutex_lock(tsd_tsdn(tsd), &t0->mtx);
-		pthread_cond_signal(&t0->cond);
+		os_cond_signal(&t0->cond);
 		malloc_mutex_unlock(tsd_tsdn(tsd), &t0->mtx);
 
 		return false;
@@ -813,7 +784,7 @@ background_thread_wakeup_early(
 	    && nstime_ns(remaining_sleep) < BACKGROUND_THREAD_MIN_INTERVAL_NS) {
 		return;
 	}
-	pthread_cond_signal(&info->cond);
+	os_cond_signal(&info->cond);
 }
 
 void
@@ -857,8 +828,8 @@ background_thread_postfork_child(tsdn_t *tsdn) {
 		background_thread_info_t *info = &background_thread_info[i];
 		malloc_mutex_lock(tsdn, &info->mtx);
 		info->state = background_thread_stopped;
-		int ret = background_thread_cond_init(&info->cond);
+		bool ret = os_cond_init(&info->cond);
-		assert(ret == 0);
+		assert(!ret);
 		background_thread_info_init(tsdn, info);
 		malloc_mutex_unlock(tsdn, &info->mtx);
 	}
@@ -975,7 +946,7 @@ background_thread_boot1(tsdn_t *tsdn, base_t *base) {
 		        malloc_mutex_address_ordered)) {
 			return true;
 		}
-		if (background_thread_cond_init(&info->cond)) {
+		if (os_cond_init(&info->cond)) {
 			return true;
 		}
 		malloc_mutex_lock(tsdn, &info->mtx);

src/jemalloc_init.c

@@ -55,10 +55,14 @@ malloc_initializer_set(void) {
 }
 
 /* Used to avoid initialization races. */
-#ifdef _WIN32
+#if !OS_MUTEX_HAS_STATIC_INIT
-#	if _WIN32_WINNT >= 0x0600
+/*
-static malloc_mutex_t init_lock = SRWLOCK_INIT;
+ * This target's os_mutex_t has no static initializer (currently: pre-Vista
-#	else
+ * Windows CRITICAL_SECTION; see os/windows/mutex.h), so MALLOC_MUTEX_
+ * INITIALIZER is empty -- init_lock needs its own lazy-init workaround
+ * instead, being the one static malloc_mutex_t that must be usable before
+ * any other code runs.
+ */
 static malloc_mutex_t init_lock;
 static bool           init_lock_initialized = false;
 
@@ -82,12 +86,11 @@ _init_init_lock(void) {
 	init_lock_initialized = true;
 }
 
-#		ifdef _MSC_VER
+#	ifdef _MSC_VER
-#			pragma section(".CRT$XCU", read)
+#		pragma section(".CRT$XCU", read)
 JEMALLOC_SECTION(".CRT$XCU")
 JEMALLOC_ATTR(used)
 static const void(WINAPI *init_init_lock)(void) = _init_init_lock;
-#		endif
 #	endif
 #else
 static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER;
@@ -599,7 +602,7 @@ malloc_init_hard(void) {
 	 */
 	assert(SC_NTINY == 0 || SC_LG_TINY_MAXCLASS <= SC_LG_LARGE_MINCLASS);
 
-#if defined(_WIN32) && _WIN32_WINNT < 0x0600
+#if !OS_MUTEX_HAS_STATIC_INIT
 	_init_init_lock();
 #endif
 	malloc_mutex_lock(TSDN_NULL, &init_lock);

src/mutex.c

@@ -5,10 +5,6 @@
 #include "jemalloc/internal/mutex.h"
 #include "jemalloc/internal/spin.h"
 
-#if defined(_WIN32) && !defined(_CRT_SPINCOUNT)
-#	define _CRT_SPINCOUNT 4000
-#endif
-
 /*
  * Based on benchmark results, a fixed spin with this amount of retries works
  * well for our critical sections.
@@ -141,18 +137,17 @@ bool
 malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank,
     malloc_mutex_lock_order_t lock_order) {
 	mutex_prof_data_init(&mutex->prof_data);
-#ifdef _WIN32
+#ifdef JEMALLOC_MUTEX_INIT_CB
-#	if _WIN32_WINNT >= 0x0600
+	/*
-	InitializeSRWLock(&mutex->lock);
+	 * Targets platforms with _pthread_mutex_init_calloc_cb(), which is
-#	else
+	 * currently just FreeBSD/libthr: its pthread_mutex_init() can calloc()
-	if (!InitializeCriticalSectionAndSpinCount(
+	 * internally, which would recurse back into jemalloc before it is safe
-	        &mutex->lock, _CRT_SPINCOUNT)) {
+	 * safe to allocate.  Until malloc_mutex_boot() runs, defer real
-		return true;
+	 * initialization by queuing the mutex here instead; malloc_mutex_boot()
-	}
+	 * later drains the queue, calling _pthread_mutex_init_calloc_cb() with
-#	endif
+	 * bootstrap_calloc (jemalloc's own early-bootstrap allocation path) as
-#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
+	 * the callback, once that's safe to use.
-	mutex->lock = OS_UNFAIR_LOCK_INIT;
+	 */
-#elif (defined(JEMALLOC_MUTEX_INIT_CB))
 	if (postpone_init) {
 		mutex->postponed_next = postponed_mutexes;
 		postponed_mutexes = mutex;
@@ -164,17 +159,9 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, witness_rank_t rank,
 		}
 	}
 #else
-	pthread_mutexattr_t attr;
+	if (os_mutex_init(&mutex->lock)) {
-
-	if (pthread_mutexattr_init(&attr) != 0) {
 		return true;
 	}
-	pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE);
-	if (pthread_mutex_init(&mutex->lock, &attr) != 0) {
-		pthread_mutexattr_destroy(&attr);
-		return true;
-	}
-	pthread_mutexattr_destroy(&attr);
 #endif
 	if (config_debug) {
 		mutex->lock_order = lock_order;