diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/include/jemalloc/internal/jemalloc_internal_inlines_a.h index 5f0d9d89..747fc79d 100644 --- a/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -7,6 +7,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/os.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/ticker.h" @@ -14,42 +15,7 @@ JEMALLOC_ALWAYS_INLINE malloc_cpuid_t malloc_getcpu(void) { assert(have_percpu_arena); -#if defined(_WIN32) - return GetCurrentProcessorNumber(); -#elif defined(JEMALLOC_HAVE_SCHED_GETCPU) - return (malloc_cpuid_t)sched_getcpu(); -#elif defined(__APPLE__) - /* - * No sched_getcpu() on macOS; read the CPU number like _os_cpu_number() - * does, from the low 12 bits of tpidr_el0 (arm64) or the IDT base (x86). - * The 0xfff mask is xnu's __TPIDR_CPU_NUM_MASK, kept in sync with - * _os_cpu_number in libsyscall/os/tsd.h (the kernel counterpart is - * MACHDEP_TPIDR_CPUNUM_MASK in osfmk/arm64/machine_machdep.h): - * https://github.com/apple-oss-distributions/xnu/blob/main/libsyscall/os/tsd.h - * This requires macOS 12+: on macOS 11 and earlier arm64 kept the CPU - * number in tpidrro_el0's low 3 bits instead, so it would be misread here. - * Those releases are retired by Apple, so only the current layout is handled. - */ -# if defined(__aarch64__) - uint64_t cpu; - __asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(cpu)); - return (malloc_cpuid_t)(cpu & 0xfff); -# elif defined(__x86_64__) || defined(__i386__) - struct { uintptr_t p1, p2; } idtr; - __asm__ __volatile__("sidt %0" : "=m"(idtr)); - return (malloc_cpuid_t)(idtr.p1 & 0xfff); -# else - not_reached(); - return -1; -# endif -#elif defined(JEMALLOC_HAVE_RDTSCP) - unsigned int ecx; - asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx"); - return (malloc_cpuid_t)(ecx & 0xfff); -#else - not_reached(); - return -1; -#endif + return (malloc_cpuid_t)os_cpu_current(); } /* Return the chosen arena index based on current cpu. */ diff --git a/include/jemalloc/internal/os.h b/include/jemalloc/internal/os.h index 99185907..4354fb42 100644 --- a/include/jemalloc/internal/os.h +++ b/include/jemalloc/internal/os.h @@ -43,4 +43,7 @@ /* Sigmask (POSIX background thread only) */ #include "jemalloc/internal/os/sigmask.h" +/* CPU */ +#include "jemalloc/internal/os/cpu.h" + #endif /* JEMALLOC_INTERNAL_OS_H */ diff --git a/include/jemalloc/internal/os/cpu.h b/include/jemalloc/internal/os/cpu.h new file mode 100644 index 00000000..6953a867 --- /dev/null +++ b/include/jemalloc/internal/os/cpu.h @@ -0,0 +1,34 @@ +#ifndef JEMALLOC_INTERNAL_OS_CPU_H +#define JEMALLOC_INTERNAL_OS_CPU_H + +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/os/detect.h" + +/* + * CPU interface: counts and current-CPU queries used for arena / tcache + * sizing. + * Default: posix/. Override: Windows (GetSystemInfo / + * GetCurrentProcessorNumber), Darwin (os_cpu_current() has no sched_getcpu() + * to fall back on, so it reads the CPU index directly out of a CPU register + * -- substantial enough, and different enough per-arch, to warrant its own + * file rather than an inline #ifdef in posix/cpu.h; os_cpu_ncpus() and + * os_cpu_count_is_deterministic() are identical to POSIX, so darwin/cpu.h + * reuses posix/cpu.h for those instead of duplicating them). + */ + +/* Functions required for implementation in each backend. */ +JEMALLOC_ALWAYS_INLINE unsigned os_cpu_ncpus(void); +JEMALLOC_ALWAYS_INLINE bool os_cpu_count_is_deterministic(void); +JEMALLOC_ALWAYS_INLINE int os_cpu_current(void); + +#if defined(_WIN32) +# include "jemalloc/internal/os/windows/cpu.h" +#elif defined(__APPLE__) +# include "jemalloc/internal/os/darwin/cpu.h" +#elif defined(JEMALLOC_OS_POSIX) +# include "jemalloc/internal/os/posix/cpu.h" +#else +# error "OS layer: no cpu backend for this platform; add os//cpu.h" +#endif + +#endif /* JEMALLOC_INTERNAL_OS_CPU_H */ diff --git a/include/jemalloc/internal/os/darwin/cpu.h b/include/jemalloc/internal/os/darwin/cpu.h new file mode 100644 index 00000000..6478126f --- /dev/null +++ b/include/jemalloc/internal/os/darwin/cpu.h @@ -0,0 +1,88 @@ +#ifndef JEMALLOC_INTERNAL_OS_DARWIN_CPU_H +#define JEMALLOC_INTERNAL_OS_DARWIN_CPU_H + +/* + * Darwin CPU backend. os_cpu_ncpus()/os_cpu_count_is_deterministic() are + * identical to posix/cpu.h's (macOS has no CPU_COUNT/sched_getaffinity() + * either, so both already fall through to the same sysconf() path) -- + * duplicated here rather than shared via #include, matching every other + * os//.h backend (each is self-contained; see os/darwin/mutex.h). + * os_cpu_current() is genuinely different: no sched_getcpu() on macOS, so it + * reads the CPU index directly out of a CPU register instead. + */ +#include "jemalloc/internal/jemalloc_preamble.h" + +JEMALLOC_ALWAYS_INLINE unsigned +os_cpu_ncpus(void) { + long result; + +#ifdef CPU_COUNT + { + cpu_set_t set; +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) + sched_getaffinity(0, sizeof(set), &set); +# else + pthread_getaffinity_np(pthread_self(), sizeof(set), &set); +# endif + result = CPU_COUNT(&set); + } +#else + result = sysconf(_SC_NPROCESSORS_ONLN); +#endif + return ((result == -1) ? 1 : (unsigned)result); +} + +JEMALLOC_ALWAYS_INLINE bool +os_cpu_count_is_deterministic(void) { + long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN); + long cpu_conf = sysconf(_SC_NPROCESSORS_CONF); + if (cpu_onln != cpu_conf) { + return false; + } +# if defined(CPU_COUNT) + cpu_set_t set; +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) + sched_getaffinity(0, sizeof(set), &set); +# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */ + pthread_getaffinity_np(pthread_self(), sizeof(set), &set); +# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */ + long cpu_affinity = CPU_COUNT(&set); + if (cpu_affinity != cpu_conf) { + return false; + } +# endif /* CPU_COUNT */ + return true; +} + +JEMALLOC_ALWAYS_INLINE int +os_cpu_current(void) { +#if defined(JEMALLOC_HAVE_SCHED_GETCPU) + return sched_getcpu(); +#else + /* + * No sched_getcpu() on macOS; read the CPU number like _os_cpu_number() + * does, from the low 12 bits of tpidr_el0 (arm64) or the IDT base (x86). + * The 0xfff mask is xnu's __TPIDR_CPU_NUM_MASK, kept in sync with + * _os_cpu_number in libsyscall/os/tsd.h (the kernel counterpart is + * MACHDEP_TPIDR_CPUNUM_MASK in osfmk/arm64/machine_machdep.h): + * https://github.com/apple-oss-distributions/xnu/blob/main/libsyscall/os/tsd.h + * This requires macOS 12+: on macOS 11 and earlier arm64 kept the CPU + * number in tpidrro_el0's low 3 bits instead, so it would be misread here. + * Those releases are retired by Apple, so only the current layout is handled. + */ +# if defined(__aarch64__) + uint64_t cpu; + __asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(cpu)); + return (int)(cpu & 0xfff); +# elif defined(__x86_64__) || defined(__i386__) + struct { uintptr_t p1, p2; } idtr; + __asm__ __volatile__("sidt %0" : "=m"(idtr)); + return (int)(idtr.p1 & 0xfff); +# else + not_reached(); + return -1; +# endif +#endif +} + +#endif /* JEMALLOC_INTERNAL_OS_DARWIN_CPU_H */ diff --git a/include/jemalloc/internal/os/posix/cpu.h b/include/jemalloc/internal/os/posix/cpu.h new file mode 100644 index 00000000..ac81a4bd --- /dev/null +++ b/include/jemalloc/internal/os/posix/cpu.h @@ -0,0 +1,87 @@ +#ifndef JEMALLOC_INTERNAL_OS_POSIX_CPU_H +#define JEMALLOC_INTERNAL_OS_POSIX_CPU_H + +#include "jemalloc/internal/jemalloc_preamble.h" + +JEMALLOC_ALWAYS_INLINE unsigned +os_cpu_ncpus(void) { + long result; + +#ifdef CPU_COUNT + /* + * glibc >= 2.6 has the CPU_COUNT macro. + * + * glibc's sysconf() uses isspace(). glibc allocates for the first time + * *before* setting up the isspace tables. Therefore we need a + * different method to get the number of CPUs. + * + * The getaffinity approach is also preferred when only a subset of CPUs + * is available, to avoid using more arenas than necessary. + */ + { +# if defined(__FreeBSD__) || defined(__DragonFly__) + cpuset_t set; +# else + cpu_set_t set; +# endif +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) + sched_getaffinity(0, sizeof(set), &set); +# else + pthread_getaffinity_np(pthread_self(), sizeof(set), &set); +# endif + result = CPU_COUNT(&set); + } +#else + result = sysconf(_SC_NPROCESSORS_ONLN); +#endif + return ((result == -1) ? 1 : (unsigned)result); +} + +/* + * Ensure that number of CPUs is determistinc, i.e. it is the same based on: + * - sched_getaffinity() + * - _SC_NPROCESSORS_ONLN + * - _SC_NPROCESSORS_CONF + * Since otherwise tricky things is possible with percpu arenas in use. + */ +JEMALLOC_ALWAYS_INLINE bool +os_cpu_count_is_deterministic(void) { + long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN); + long cpu_conf = sysconf(_SC_NPROCESSORS_CONF); + if (cpu_onln != cpu_conf) { + return false; + } +# if defined(CPU_COUNT) +# if defined(__FreeBSD__) || defined(__DragonFly__) + cpuset_t set; +# else + cpu_set_t set; +# endif /* __FreeBSD__ */ +# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) + sched_getaffinity(0, sizeof(set), &set); +# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */ + pthread_getaffinity_np(pthread_self(), sizeof(set), &set); +# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */ + long cpu_affinity = CPU_COUNT(&set); + if (cpu_affinity != cpu_conf) { + return false; + } +# endif /* CPU_COUNT */ + return true; +} + +JEMALLOC_ALWAYS_INLINE int +os_cpu_current(void) { +#if defined(JEMALLOC_HAVE_SCHED_GETCPU) + return sched_getcpu(); +#elif defined(JEMALLOC_HAVE_RDTSCP) + unsigned int ecx; + asm volatile("rdtscp" : "=c"(ecx)::"eax", "edx"); + return (int)(ecx & 0xfff); +#else + not_reached(); + return -1; +#endif +} + +#endif /* JEMALLOC_INTERNAL_OS_POSIX_CPU_H */ diff --git a/include/jemalloc/internal/os/windows/cpu.h b/include/jemalloc/internal/os/windows/cpu.h new file mode 100644 index 00000000..29f8254f --- /dev/null +++ b/include/jemalloc/internal/os/windows/cpu.h @@ -0,0 +1,23 @@ +#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_CPU_H +#define JEMALLOC_INTERNAL_OS_WINDOWS_CPU_H + +#include "jemalloc/internal/jemalloc_preamble.h" + +JEMALLOC_ALWAYS_INLINE unsigned +os_cpu_ncpus(void) { + SYSTEM_INFO si; + GetSystemInfo(&si); + return (unsigned)si.dwNumberOfProcessors; +} + +JEMALLOC_ALWAYS_INLINE bool +os_cpu_count_is_deterministic(void) { + return true; +} + +JEMALLOC_ALWAYS_INLINE int +os_cpu_current(void) { + return (int)GetCurrentProcessorNumber(); +} + +#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_CPU_H */ diff --git a/src/jemalloc_init.c b/src/jemalloc_init.c index 27ebc3fe..8f2ff4bc 100644 --- a/src/jemalloc_init.c +++ b/src/jemalloc_init.c @@ -16,6 +16,7 @@ #include "jemalloc/internal/jemalloc_init.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/os.h" #include "jemalloc/internal/san.h" #include "jemalloc/internal/sc.h" #include "jemalloc/internal/spin.h" @@ -124,8 +125,6 @@ malloc_slow_flag_init(void) { } static void stats_print_atexit(void); -static unsigned malloc_ncpus(void); -static bool malloc_cpu_count_is_deterministic(void); static bool malloc_init_hard_needed(void) { @@ -243,7 +242,7 @@ malloc_init_hard_a0_locked(void) { experimental_thread_events_boot(); /* * Create enough scaffolding to allow recursive allocation in - * malloc_ncpus(). + * os_cpu_ncpus(). */ narenas_auto_set(1); manual_arena_base_set(narenas_auto + 1); @@ -317,90 +316,15 @@ stats_print_atexit(void) { je_malloc_stats_print(NULL, NULL, opt_stats_print_opts); } -static unsigned -malloc_ncpus(void) { - long result; - -#ifdef _WIN32 - SYSTEM_INFO si; - GetSystemInfo(&si); - result = si.dwNumberOfProcessors; -#elif defined(CPU_COUNT) - /* - * glibc >= 2.6 has the CPU_COUNT macro. - * - * glibc's sysconf() uses isspace(). glibc allocates for the first time - * *before* setting up the isspace tables. Therefore we need a - * different method to get the number of CPUs. - * - * The getaffinity approach is also preferred when only a subset of CPUs - * is available, to avoid using more arenas than necessary. - */ - { -# if defined(__FreeBSD__) || defined(__DragonFly__) - cpuset_t set; -# else - cpu_set_t set; -# endif -# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) - sched_getaffinity(0, sizeof(set), &set); -# else - pthread_getaffinity_np(pthread_self(), sizeof(set), &set); -# endif - result = CPU_COUNT(&set); - } -#else - result = sysconf(_SC_NPROCESSORS_ONLN); -#endif - return ((result == -1) ? 1 : (unsigned)result); -} - -/* - * Ensure that number of CPUs is determistinc, i.e. it is the same based on: - * - sched_getaffinity() - * - _SC_NPROCESSORS_ONLN - * - _SC_NPROCESSORS_CONF - * Since otherwise tricky things is possible with percpu arenas in use. - */ -static bool -malloc_cpu_count_is_deterministic(void) { -#ifdef _WIN32 - return true; -#else - long cpu_onln = sysconf(_SC_NPROCESSORS_ONLN); - long cpu_conf = sysconf(_SC_NPROCESSORS_CONF); - if (cpu_onln != cpu_conf) { - return false; - } -# if defined(CPU_COUNT) -# if defined(__FreeBSD__) || defined(__DragonFly__) - cpuset_t set; -# else - cpu_set_t set; -# endif /* __FreeBSD__ */ -# if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) - sched_getaffinity(0, sizeof(set), &set); -# else /* !JEMALLOC_HAVE_SCHED_SETAFFINITY */ - pthread_getaffinity_np(pthread_self(), sizeof(set), &set); -# endif /* JEMALLOC_HAVE_SCHED_SETAFFINITY */ - long cpu_affinity = CPU_COUNT(&set); - if (cpu_affinity != cpu_conf) { - return false; - } -# endif /* CPU_COUNT */ - return true; -#endif -} - /* Initialize data structures which may trigger recursive allocation. */ static bool malloc_init_hard_recursible(void) { malloc_init_state = malloc_init_recursible; - ncpus = malloc_ncpus(); + ncpus = os_cpu_ncpus(); if (opt_percpu_arena != percpu_arena_disabled) { bool cpu_count_is_deterministic = - malloc_cpu_count_is_deterministic(); + os_cpu_count_is_deterministic(); if (!cpu_count_is_deterministic) { /* * If # of CPU is not deterministic, and narenas not