diff --git a/Makefile.in b/Makefile.in index ac8c51ff..bb0f5ad5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -152,6 +152,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/safety_check.c \ $(srcroot)src/sc.c \ $(srcroot)src/sec.c \ + $(srcroot)src/spin_delay_arm.c \ $(srcroot)src/stats.c \ $(srcroot)src/sz.c \ $(srcroot)src/tcache.c \ diff --git a/include/jemalloc/internal/spin.h b/include/jemalloc/internal/spin.h index 87c400d5..127edfc3 100644 --- a/include/jemalloc/internal/spin.h +++ b/include/jemalloc/internal/spin.h @@ -2,6 +2,7 @@ #define JEMALLOC_INTERNAL_SPIN_H #include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/spin_delay_arm.h" #define SPIN_INITIALIZER {0U} @@ -11,7 +12,9 @@ typedef struct { static inline void spin_cpu_spinwait(void) { -# if HAVE_CPU_SPINWAIT +# if defined(__linux__) && (defined(__aarch64__) || defined(__arm64__)) + spin_delay_arm(); +# elif HAVE_CPU_SPINWAIT CPU_SPINWAIT; # else volatile int x = 0; diff --git a/include/jemalloc/internal/spin_delay_arm.h b/include/jemalloc/internal/spin_delay_arm.h new file mode 100644 index 00000000..3ea86c6c --- /dev/null +++ b/include/jemalloc/internal/spin_delay_arm.h @@ -0,0 +1,21 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include + +/* Global variable to track SB support, declared as extern to be defined in one TU */ +extern _Atomic int arm_has_sb_instruction; + +/* Constructor function declaration - implementation in spin_delay_arm.c */ +__attribute__((constructor)) +void detect_arm_sb_support(void); + +/* Use SB instruction if available, otherwise ISB */ +static inline void +spin_delay_arm(void) { + if (__builtin_expect(arm_has_sb_instruction == 1, 1)) { + /* SB instruction encoding */ + asm volatile(".inst 0xd50330ff \n"); + } else { + /* ISB instruction */ + asm volatile("isb; \n"); + } +} diff --git a/src/spin_delay_arm.c b/src/spin_delay_arm.c new file mode 100644 index 00000000..975023b0 --- /dev/null +++ b/src/spin_delay_arm.c @@ -0,0 +1,27 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/spin_delay_arm.h" +#include + +#if defined(__linux__) && (defined(__aarch64__) || defined(__arm64__)) +#include + +/* Define HWCAP_SB if not already defined in system headers */ +#ifndef HWCAP_SB +#define HWCAP_SB (1ULL << 56) /* Speculation Barrier */ +#endif // HWCAP_SB +#endif // __linux__ && (defined(__aarch64__) || defined(__arm64__)) + +/* Global variable to track SB support, defined here to avoid multiple definitions */ +_Atomic int arm_has_sb_instruction = ATOMIC_VAR_INIT(0); + +/* Constructor function to detect hardware capabilities at program startup */ +__attribute__((constructor)) +void +detect_arm_sb_support(void) { +#if defined(__linux__) && (defined(__aarch64__) || defined(__arm64__)) + /* Check if SB instruction is supported */ + if (getauxval(AT_HWCAP) & HWCAP_SB) { + atomic_store_explicit(&arm_has_sb_instruction, 1, memory_order_release); + } +#endif +} \ No newline at end of file