diff --git a/include/jemalloc/internal/prof_inlines_a.h b/include/jemalloc/internal/prof_inlines_a.h index d77635a8..d0d29685 100644 --- a/include/jemalloc/internal/prof_inlines_a.h +++ b/include/jemalloc/internal/prof_inlines_a.h @@ -22,15 +22,16 @@ prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { * avoids rate-limiting allocation. */ #ifdef JEMALLOC_ATOMIC_U64 + a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); do { - a0 = atomic_read_u64(&prof_accum->accumbytes); a1 = a0 + accumbytes; assert(a1 >= a0); overflow = (a1 >= prof_interval); if (overflow) { a1 %= prof_interval; } - } while (atomic_cas_u64(&prof_accum->accumbytes, a0, a1)); + } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, + a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); #else malloc_mutex_lock(tsdn, &prof_accum->mtx); a0 = prof_accum->accumbytes; @@ -57,11 +58,12 @@ prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { */ uint64_t a0, a1; #ifdef JEMALLOC_ATOMIC_U64 + a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); do { - a0 = atomic_read_u64(&prof_accum->accumbytes); a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : 0; - } while (atomic_cas_u64(&prof_accum->accumbytes, a0, a1)); + } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, + a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); #else malloc_mutex_lock(tsdn, &prof_accum->mtx); a0 = prof_accum->accumbytes; diff --git a/include/jemalloc/internal/prof_structs.h b/include/jemalloc/internal/prof_structs.h index afff6aa5..fba8c295 100644 --- a/include/jemalloc/internal/prof_structs.h +++ b/include/jemalloc/internal/prof_structs.h @@ -18,8 +18,10 @@ typedef struct { struct prof_accum_s { #ifndef JEMALLOC_ATOMIC_U64 malloc_mutex_t mtx; -#endif uint64_t accumbytes; +#else + atomic_u64_t accumbytes; +#endif }; struct prof_cnt_s { diff --git a/src/prof.c b/src/prof.c index 4e83ae3f..ce02d99c 100644 --- a/src/prof.c +++ b/src/prof.c @@ -1758,8 +1758,10 @@ prof_accum_init(tsdn_t *tsdn, prof_accum_t *prof_accum) { WITNESS_RANK_PROF_ACCUM)) { return true; } -#endif prof_accum->accumbytes = 0; +#else + atomic_store_u64(&prof_accum->accumbytes, 0, ATOMIC_RELAXED); +#endif return false; }