diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index d97b6a7d..7b16d229 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -39,7 +39,7 @@ void arena_extent_ralloc_large_shrink(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t oldsize); void arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t oldsize); -ssize_t arena_decay_time_get(tsdn_t *tsdn, arena_t *arena); +ssize_t arena_decay_time_get(arena_t *arena); bool arena_decay_time_set(tsdn_t *tsdn, arena_t *arena, ssize_t decay_time); void arena_purge(tsdn_t *tsdn, arena_t *arena, bool all); void arena_maybe_purge(tsdn_t *tsdn, arena_t *arena); diff --git a/include/jemalloc/internal/arena_structs_b.h b/include/jemalloc/internal/arena_structs_b.h index 92f1e41f..49fdd17d 100644 --- a/include/jemalloc/internal/arena_structs_b.h +++ b/include/jemalloc/internal/arena_structs_b.h @@ -36,15 +36,22 @@ struct arena_bin_info_s { bitmap_info_t bitmap_info; }; +typedef union { + size_t u; /* Used for atomic operations. */ + ssize_t s; /* Time may be negative (means "never"). */ +} arena_decay_time_t; + struct arena_decay_s { - /* Synchronizes all fields. */ + /* Synchronizes all non-atomic fields. */ malloc_mutex_t mtx; /* * Approximate time in seconds from the creation of a set of unused * dirty pages until an equivalent set of unused dirty pages is purged * and/or reused. + * + * Synchronization: atomic. */ - ssize_t time; + arena_decay_time_t time; /* time / SMOOTHSTEP_NSTEPS. */ nstime_t interval; /* diff --git a/src/arena.c b/src/arena.c index 18b49312..9f395769 100644 --- a/src/arena.c +++ b/src/arena.c @@ -157,7 +157,7 @@ arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *decay_time, size_t *nactive, size_t *ndirty) { *nthreads += arena_nthreads_get(arena, false); *dss = dss_prec_names[arena_dss_prec_get(arena)]; - *decay_time = arena_decay_time_get(tsdn, arena); + *decay_time = arena_decay_time_get(arena); *nactive += atomic_read_zu(&arena->nactive); *ndirty += extents_npages_get(&arena->extents_cached); } @@ -491,6 +491,20 @@ arena_extent_ralloc_large_expand(tsdn_t *tsdn, arena_t *arena, extent_t *extent, arena_nactive_add(arena, udiff >> LG_PAGE); } +static ssize_t +arena_decay_time_read(arena_t *arena) { + arena_decay_time_t dt; + dt.u = atomic_read_zu(&arena->decay.time.u); + return dt.s; +} + +static void +arena_decay_time_write(arena_t *arena, ssize_t decay_time) { + arena_decay_time_t dt; + dt.s = decay_time; + atomic_write_zu(&arena->decay.time.u, dt.u); +} + static void arena_decay_deadline_init(arena_t *arena) { /* @@ -499,7 +513,7 @@ arena_decay_deadline_init(arena_t *arena) { */ nstime_copy(&arena->decay.deadline, &arena->decay.epoch); nstime_add(&arena->decay.deadline, &arena->decay.interval); - if (arena->decay.time > 0) { + if (arena_decay_time_read(arena) > 0) { nstime_t jitter; nstime_init(&jitter, prng_range_u64(&arena->decay.jitter_state, @@ -615,7 +629,7 @@ arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, const nstime_t *time) { static void arena_decay_reinit(arena_t *arena, ssize_t decay_time) { - arena->decay.time = decay_time; + arena_decay_time_write(arena, decay_time); if (decay_time > 0) { nstime_init2(&arena->decay.interval, decay_time, 0); nstime_idivide(&arena->decay.interval, SMOOTHSTEP_NSTEPS); @@ -650,14 +664,8 @@ arena_decay_time_valid(ssize_t decay_time) { } ssize_t -arena_decay_time_get(tsdn_t *tsdn, arena_t *arena) { - ssize_t decay_time; - - malloc_mutex_lock(tsdn, &arena->decay.mtx); - decay_time = arena->decay.time; - malloc_mutex_unlock(tsdn, &arena->decay.mtx); - - return decay_time; +arena_decay_time_get(arena_t *arena) { + return arena_decay_time_read(arena); } bool @@ -687,8 +695,9 @@ arena_maybe_purge(tsdn_t *tsdn, arena_t *arena) { malloc_mutex_assert_owner(tsdn, &arena->decay.mtx); /* Purge all or nothing if the option is disabled. */ - if (arena->decay.time <= 0) { - if (arena->decay.time == 0) { + ssize_t decay_time = arena_decay_time_read(arena); + if (decay_time <= 0) { + if (decay_time == 0) { arena_purge_to_limit(tsdn, arena, 0); } return; diff --git a/src/ctl.c b/src/ctl.c index 7ec8ff2a..83e9e93e 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1797,7 +1797,7 @@ arena_i_decay_time_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } if (oldp != NULL && oldlenp != NULL) { - size_t oldval = arena_decay_time_get(tsd_tsdn(tsd), arena); + size_t oldval = arena_decay_time_get(arena); READ(oldval, ssize_t); } if (newp != NULL) {