romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-08-02 00:20:31 +03:00
Code Issues Projects Releases Packages Wiki Activity

Move CPU facility to OS layer

Browse source 
Migrates CPU-count detection, current-CPU-index, and CPU deterministic
queries onto the OS layer.
This commit is contained in:
guangli-dai 2026-07-27 17:29:50 -07:00 • committed by Guangli Dai
parent 192271fb82
commit 85599c8b12
7 changed files with 241 additions and 116 deletions
Download patch file Download diff file Expand all files Collapse all files

38
include/jemalloc/internal/jemalloc_internal_inlines_a.h
View file

@ -7,6 +7,7 @@
#include "jemalloc/internal/atomic.h"
#include "jemalloc/internal/bit_util.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/os.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/tcache.h"
#include "jemalloc/internal/ticker.h"
@ -14,42 +15,7 @@
JEMALLOC_ALWAYS_INLINE malloc_cpuid_t
malloc_getcpu(void) {
assert(have_percpu_arena);
#if defined(_WIN32)
return GetCurrentProcessorNumber();
#elif defined(JEMALLOC_HAVE_SCHED_GETCPU)
return (malloc_cpuid_t)sched_getcpu();
#elif defined(__APPLE__)
/*
* No sched_getcpu() on macOS; read the CPU number like _os_cpu_number()
* does, from the low 12 bits of tpidr_el0 (arm64) or the IDT base (x86).
* The 0xfff mask is xnu's __TPIDR_CPU_NUM_MASK, kept in sync with
* _os_cpu_number in libsyscall/os/tsd.h (the kernel counterpart is
* MACHDEP_TPIDR_CPUNUM_MASK in osfmk/arm64/machine_machdep.h):
* https://github.com/apple-oss-distributions/xnu/blob/main/libsyscall/os/tsd.h
* This requires macOS 12+: on macOS 11 and earlier arm64 kept the CPU
* number in tpidrro_el0's low 3 bits instead, so it would be misread here.
* Those releases are retired by Apple, so only the current layout is handled.
*/
# if defined(__aarch64__)
uint64_t cpu;
__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(cpu));
return (malloc_cpuid_t)(cpu & 0xfff);
# elif defined(__x86_64__) || defined(__i386__)
struct { uintptr_t p1, p2; } idtr;
__asm__ __volatile__("sidt %0" : "=m"(idtr));
return (malloc_cpuid_t)(idtr.p1 & 0xfff);
# else
not_reached();
return -1;
# endif
#elif defined(JEMALLOC_HAVE_RDTSCP)
unsigned int ecx;
asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx");
return (malloc_cpuid_t)(ecx & 0xfff);
#else
not_reached();
return -1;
#endif
return (malloc_cpuid_t)os_cpu_current();
}
/* Return the chosen arena index based on current cpu. */

3
include/jemalloc/internal/os.h
View file

@ -43,4 +43,7 @@
/* Sigmask (POSIX background thread only) */
#include "jemalloc/internal/os/sigmask.h"
/* CPU */
#include "jemalloc/internal/os/cpu.h"
#endif /* JEMALLOC_INTERNAL_OS_H */

34
include/jemalloc/internal/os/cpu.h Normal file
View file

@ -0,0 +1,34 @@
#ifndef JEMALLOC_INTERNAL_OS_CPU_H
#define JEMALLOC_INTERNAL_OS_CPU_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/os/detect.h"
/*
* CPU interface: counts and current-CPU queries used for arena / tcache
* sizing.
* Default: posix/. Override: Windows (GetSystemInfo /
* GetCurrentProcessorNumber), Darwin (os_cpu_current() has no sched_getcpu()
* to fall back on, so it reads the CPU index directly out of a CPU register
* -- substantial enough, and different enough per-arch, to warrant its own
* file rather than an inline #ifdef in posix/cpu.h; os_cpu_ncpus() and
* os_cpu_count_is_deterministic() are identical to POSIX, so darwin/cpu.h
* reuses posix/cpu.h for those instead of duplicating them).
*/
/* Functions required for implementation in each backend. */
JEMALLOC_ALWAYS_INLINE unsigned os_cpu_ncpus(void);
JEMALLOC_ALWAYS_INLINE bool os_cpu_count_is_deterministic(void);
JEMALLOC_ALWAYS_INLINE int os_cpu_current(void);
#if defined(_WIN32)
# include "jemalloc/internal/os/windows/cpu.h"
#elif defined(__APPLE__)
# include "jemalloc/internal/os/darwin/cpu.h"
#elif defined(JEMALLOC_OS_POSIX)
# include "jemalloc/internal/os/posix/cpu.h"
#else
# error "OS layer: no cpu backend for this platform; add os/<os>/cpu.h"
#endif
#endif /* JEMALLOC_INTERNAL_OS_CPU_H */

88
include/jemalloc/internal/os/darwin/cpu.h Normal file
View file

@ -0,0 +1,88 @@
#ifndef JEMALLOC_INTERNAL_OS_DARWIN_CPU_H
#define JEMALLOC_INTERNAL_OS_DARWIN_CPU_H
/*
* Darwin CPU backend. os_cpu_ncpus()/os_cpu_count_is_deterministic() are
* identical to posix/cpu.h's (macOS has no CPU_COUNT/sched_getaffinity()
* either, so both already fall through to the same sysconf() path) --
* duplicated here rather than shared via #include, matching every other
* os/<os>/<module>.h backend (each is self-contained; see os/darwin/mutex.h).
* os_cpu_current() is genuinely different: no sched_getcpu() on macOS, so it
* reads the CPU index directly out of a CPU register instead.
*/
#include "jemalloc/internal/jemalloc_preamble.h"
JEMALLOC_ALWAYS_INLINE unsigned
os_cpu_ncpus(void) {
long result;
#ifdef CPU_COUNT
{
cpu_set_t set;
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif
result = CPU_COUNT(&set);
}
#else
result = sysconf(_SC_NPROCESSORS_ONLN);
#endif
return ((result == -1) ? 1 : (unsigned)result);
}
JEMALLOC_ALWAYS_INLINE bool
os_cpu_count_is_deterministic(void) {
long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
if (cpu_onln != cpu_conf) {
return false;
}
# if defined(CPU_COUNT)
cpu_set_t set;
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
long cpu_affinity = CPU_COUNT(&set);
if (cpu_affinity != cpu_conf) {
return false;
}
# endif /* CPU_COUNT */
return true;
}
JEMALLOC_ALWAYS_INLINE int
os_cpu_current(void) {
#if defined(JEMALLOC_HAVE_SCHED_GETCPU)
return sched_getcpu();
#else
/*
* No sched_getcpu() on macOS; read the CPU number like _os_cpu_number()
* does, from the low 12 bits of tpidr_el0 (arm64) or the IDT base (x86).
* The 0xfff mask is xnu's __TPIDR_CPU_NUM_MASK, kept in sync with
* _os_cpu_number in libsyscall/os/tsd.h (the kernel counterpart is
* MACHDEP_TPIDR_CPUNUM_MASK in osfmk/arm64/machine_machdep.h):
* https://github.com/apple-oss-distributions/xnu/blob/main/libsyscall/os/tsd.h
* This requires macOS 12+: on macOS 11 and earlier arm64 kept the CPU
* number in tpidrro_el0's low 3 bits instead, so it would be misread here.
* Those releases are retired by Apple, so only the current layout is handled.
*/
# if defined(__aarch64__)
uint64_t cpu;
__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(cpu));
return (int)(cpu & 0xfff);
# elif defined(__x86_64__) || defined(__i386__)
struct { uintptr_t p1, p2; } idtr;
__asm__ __volatile__("sidt %0" : "=m"(idtr));
return (int)(idtr.p1 & 0xfff);
# else
not_reached();
return -1;
# endif
#endif
}
#endif /* JEMALLOC_INTERNAL_OS_DARWIN_CPU_H */

87
include/jemalloc/internal/os/posix/cpu.h Normal file
View file

@ -0,0 +1,87 @@
#ifndef JEMALLOC_INTERNAL_OS_POSIX_CPU_H
#define JEMALLOC_INTERNAL_OS_POSIX_CPU_H
#include "jemalloc/internal/jemalloc_preamble.h"
JEMALLOC_ALWAYS_INLINE unsigned
os_cpu_ncpus(void) {
long result;
#ifdef CPU_COUNT
/*
* glibc >= 2.6 has the CPU_COUNT macro.
*
* glibc's sysconf() uses isspace(). glibc allocates for the first time
* *before* setting up the isspace tables. Therefore we need a
* different method to get the number of CPUs.
*
* The getaffinity approach is also preferred when only a subset of CPUs
* is available, to avoid using more arenas than necessary.
*/
{
# if defined(__FreeBSD__) || defined(__DragonFly__)
cpuset_t set;
# else
cpu_set_t set;
# endif
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif
result = CPU_COUNT(&set);
}
#else
result = sysconf(_SC_NPROCESSORS_ONLN);
#endif
return ((result == -1) ? 1 : (unsigned)result);
}
/*
* Ensure that number of CPUs is determistinc, i.e. it is the same based on:
* - sched_getaffinity()
* - _SC_NPROCESSORS_ONLN
* - _SC_NPROCESSORS_CONF
* Since otherwise tricky things is possible with percpu arenas in use.
*/
JEMALLOC_ALWAYS_INLINE bool
os_cpu_count_is_deterministic(void) {
long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
if (cpu_onln != cpu_conf) {
return false;
}
# if defined(CPU_COUNT)
# if defined(__FreeBSD__) || defined(__DragonFly__)
cpuset_t set;
# else
cpu_set_t set;
# endif /* __FreeBSD__ */
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
long cpu_affinity = CPU_COUNT(&set);
if (cpu_affinity != cpu_conf) {
return false;
}
# endif /* CPU_COUNT */
return true;
}
JEMALLOC_ALWAYS_INLINE int
os_cpu_current(void) {
#if defined(JEMALLOC_HAVE_SCHED_GETCPU)
return sched_getcpu();
#elif defined(JEMALLOC_HAVE_RDTSCP)
unsigned int ecx;
asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx");
return (int)(ecx & 0xfff);
#else
not_reached();
return -1;
#endif
}
#endif /* JEMALLOC_INTERNAL_OS_POSIX_CPU_H */

23
include/jemalloc/internal/os/windows/cpu.h Normal file
View file

@ -0,0 +1,23 @@
#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_CPU_H
#define JEMALLOC_INTERNAL_OS_WINDOWS_CPU_H
#include "jemalloc/internal/jemalloc_preamble.h"
JEMALLOC_ALWAYS_INLINE unsigned
os_cpu_ncpus(void) {
SYSTEM_INFO si;
GetSystemInfo(&si);
return (unsigned)si.dwNumberOfProcessors;
}
JEMALLOC_ALWAYS_INLINE bool
os_cpu_count_is_deterministic(void) {
return true;
}
JEMALLOC_ALWAYS_INLINE int
os_cpu_current(void) {
return (int)GetCurrentProcessorNumber();
}
#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_CPU_H */

84
src/jemalloc_init.c
View file

@ -16,6 +16,7 @@
#include "jemalloc/internal/jemalloc_init.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/os.h"
#include "jemalloc/internal/san.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/spin.h"
@ -124,8 +125,6 @@ malloc_slow_flag_init(void) {
}
static void stats_print_atexit(void);
static unsigned malloc_ncpus(void);
static bool malloc_cpu_count_is_deterministic(void);
static bool
malloc_init_hard_needed(void) {
@ -243,7 +242,7 @@ malloc_init_hard_a0_locked(void) {
experimental_thread_events_boot();
/*
* Create enough scaffolding to allow recursive allocation in
* malloc_ncpus().
* os_cpu_ncpus().
*/
narenas_auto_set(1);
manual_arena_base_set(narenas_auto + 1);
@ -317,90 +316,15 @@ stats_print_atexit(void) {
je_malloc_stats_print(NULL, NULL, opt_stats_print_opts);
}
static unsigned
malloc_ncpus(void) {
long result;
#ifdef _WIN32
SYSTEM_INFO si;
GetSystemInfo(&si);
result = si.dwNumberOfProcessors;
#elif defined(CPU_COUNT)
/*
* glibc >= 2.6 has the CPU_COUNT macro.
*
* glibc's sysconf() uses isspace(). glibc allocates for the first time
* *before* setting up the isspace tables. Therefore we need a
* different method to get the number of CPUs.
*
* The getaffinity approach is also preferred when only a subset of CPUs
* is available, to avoid using more arenas than necessary.
*/
{
# if defined(__FreeBSD__) || defined(__DragonFly__)
cpuset_t set;
# else
cpu_set_t set;
# endif
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif
result = CPU_COUNT(&set);
}
#else
result = sysconf(_SC_NPROCESSORS_ONLN);
#endif
return ((result == -1) ? 1 : (unsigned)result);
}
/*
* Ensure that number of CPUs is determistinc, i.e. it is the same based on:
* - sched_getaffinity()
* - _SC_NPROCESSORS_ONLN
* - _SC_NPROCESSORS_CONF
* Since otherwise tricky things is possible with percpu arenas in use.
*/
static bool
malloc_cpu_count_is_deterministic(void) {
#ifdef _WIN32
return true;
#else
long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN);
long cpu_conf = sysconf(_SC_NPROCESSORS_CONF);
if (cpu_onln != cpu_conf) {
return false;
}
# if defined(CPU_COUNT)
# if defined(__FreeBSD__) || defined(__DragonFly__)
cpuset_t set;
# else
cpu_set_t set;
# endif /* __FreeBSD__ */
# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY)
sched_getaffinity(0, sizeof(set), &set);
# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */
pthread_getaffinity_np(pthread_self(), sizeof(set), &set);
# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */
long cpu_affinity = CPU_COUNT(&set);
if (cpu_affinity != cpu_conf) {
return false;
}
# endif /* CPU_COUNT */
return true;
#endif
}
/* Initialize data structures which may trigger recursive allocation. */
static bool
malloc_init_hard_recursible(void) {
malloc_init_state = malloc_init_recursible;
ncpus = malloc_ncpus();
ncpus = os_cpu_ncpus();
if (opt_percpu_arena != percpu_arena_disabled) {
bool cpu_count_is_deterministic =
malloc_cpu_count_is_deterministic();
os_cpu_count_is_deterministic();
if (!cpu_count_is_deterministic) {
/*
* If # of CPU is not deterministic, and narenas not