diff --git a/Makefile.in b/Makefile.in index 7085a22a..71a34ab1 100644 --- a/Makefile.in +++ b/Makefile.in @@ -151,6 +151,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/safety_check.c \ $(srcroot)src/sc.c \ $(srcroot)src/sec.c \ + $(srcroot)src/spin_delay_arm.c \ $(srcroot)src/stats.c \ $(srcroot)src/sz.c \ $(srcroot)src/tcache.c \ diff --git a/include/jemalloc/internal/spin.h b/include/jemalloc/internal/spin.h index 87c400d5..35c3c8fe 100644 --- a/include/jemalloc/internal/spin.h +++ b/include/jemalloc/internal/spin.h @@ -2,6 +2,7 @@ #define JEMALLOC_INTERNAL_SPIN_H #include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/spin_delay_arm.h" #define SPIN_INITIALIZER {0U} @@ -11,7 +12,10 @@ typedef struct { static inline void spin_cpu_spinwait(void) { -# if HAVE_CPU_SPINWAIT +# if defined(__linux__) && (defined(__aarch64__) || defined(__arm64__)) && \ + (defined(__GNUC__) || defined(__clang__)) + spin_delay_arm(); +# elif HAVE_CPU_SPINWAIT CPU_SPINWAIT; # else volatile int x = 0; diff --git a/include/jemalloc/internal/spin_delay_arm.h b/include/jemalloc/internal/spin_delay_arm.h new file mode 100644 index 00000000..1a711afe --- /dev/null +++ b/include/jemalloc/internal/spin_delay_arm.h @@ -0,0 +1,13 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +/* Global variable to track SB support */ +extern int arm_has_sb_instruction; + +/* Use SB instruction if available, otherwise ISB */ +static inline void spin_delay_arm(void) { + if (__builtin_expect(arm_has_sb_instruction == 1, 1)) { + asm volatile(".inst 0xd50330ff \n"); /* SB instruction encoding */ + } else { + asm volatile("isb; \n"); + } +} diff --git a/src/spin_delay_arm.c b/src/spin_delay_arm.c new file mode 100644 index 00000000..ce9fdbe3 --- /dev/null +++ b/src/spin_delay_arm.c @@ -0,0 +1,15 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/spin_delay_arm.h" + +/* Initialize to 0 (false) by default */ +int arm_has_sb_instruction = 0; + +#if defined(__linux__) && (defined(__aarch64__) || defined(__arm64__)) && \ + (defined(__GNUC__) || defined(__clang__)) +#include + +__attribute__((constructor)) +void detect_arm_sb_support(void) { + arm_has_sb_instruction = (getauxval(AT_HWCAP) & HWCAP_SB) ? 1 : 0; +} +#endif