From ea9961acdbc9f7e2c95a3b55ce0ac1024af5d167 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 3 Nov 2016 21:18:50 -0700 Subject: [PATCH] Fix psz/pind edge cases. Add an "over-size" extent heap in which to store extents which exceed the maximum size class (plus cache-oblivious padding, if enabled). Remove psz2ind_clamp() and use psz2ind() instead so that trying to allocate the maximum size class can in principle succeed. In practice, this allows assertions to hold so that OOM errors can be successfully generated. --- include/jemalloc/internal/arena.h | 4 +-- .../jemalloc/internal/jemalloc_internal.h.in | 28 ++++------------ include/jemalloc/internal/private_symbols.txt | 2 -- src/arena.c | 2 +- src/extent.c | 21 ++++++------ src/jemalloc.c | 3 +- test/unit/size_classes.c | 32 ++++++++++--------- 7 files changed, 39 insertions(+), 53 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 4e20af48..ce9d8b5e 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -212,8 +212,8 @@ struct arena_s { * Heaps of extents that were previously allocated. These are used when * allocating extents, in an attempt to re-use address space. */ - extent_heap_t extents_cached[NPSIZES]; - extent_heap_t extents_retained[NPSIZES]; + extent_heap_t extents_cached[NPSIZES+1]; + extent_heap_t extents_retained[NPSIZES+1]; /* * Ring sentinel used to track unused dirty memory. Dirty memory is * managed as an LRU of cached extents. diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 0e4ffd91..85b34012 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -434,7 +434,7 @@ extern arena_t **arenas; * pind2sz_tab encodes the same information as could be computed by * pind2sz_compute(). */ -extern size_t const pind2sz_tab[NPSIZES]; +extern size_t const pind2sz_tab[NPSIZES+1]; /* * index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by index2size_compute(). @@ -516,9 +516,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/large.h" #ifndef JEMALLOC_ENABLE_INLINE -pszind_t psz2ind_impl(size_t psz, bool clamp); pszind_t psz2ind(size_t psz); -pszind_t psz2ind_clamp(size_t psz); size_t pind2sz_compute(pszind_t pind); size_t pind2sz_lookup(pszind_t pind); size_t pind2sz(pszind_t pind); @@ -544,11 +542,11 @@ ticker_t *decay_ticker_get(tsd_t *tsd, unsigned ind); #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) JEMALLOC_ALWAYS_INLINE pszind_t -psz2ind_impl(size_t psz, bool clamp) +psz2ind(size_t psz) { if (unlikely(psz > LARGE_MAXCLASS)) - return (clamp ? NPSIZES-1 : NPSIZES); + return (NPSIZES); { pszind_t x = lg_floor((psz<<1)-1); pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - @@ -567,24 +565,12 @@ psz2ind_impl(size_t psz, bool clamp) } } -JEMALLOC_INLINE pszind_t -psz2ind(size_t psz) -{ - - return (psz2ind_impl(psz, false)); -} - -JEMALLOC_INLINE pszind_t -psz2ind_clamp(size_t psz) -{ - - return (psz2ind_impl(psz, true)); -} - JEMALLOC_INLINE size_t pind2sz_compute(pszind_t pind) { + if (unlikely(pind == NPSIZES)) + return (LARGE_MAXCLASS + PAGE); { size_t grp = pind >> LG_SIZE_CLASS_GROUP; size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); @@ -614,7 +600,7 @@ JEMALLOC_INLINE size_t pind2sz(pszind_t pind) { - assert(pind < NPSIZES); + assert(pind < NPSIZES+1); return (pind2sz_lookup(pind)); } @@ -623,7 +609,7 @@ psz2u(size_t psz) { if (unlikely(psz > LARGE_MAXCLASS)) - return (0); + return (LARGE_MAXCLASS + PAGE); { size_t x = lg_floor((psz<<1)-1); size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 2e2c11d8..2949de10 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -380,8 +380,6 @@ prof_thread_active_set prof_thread_name_get prof_thread_name_set psz2ind -psz2ind_clamp -psz2ind_impl psz2u rtree_child_read rtree_child_read_hard diff --git a/src/arena.c b/src/arena.c index fd3c5531..dd8e4d9c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1696,7 +1696,7 @@ arena_new(tsdn_t *tsdn, unsigned ind) WITNESS_RANK_ARENA_LARGE)) return (NULL); - for (i = 0; i < NPSIZES; i++) { + for (i = 0; i < NPSIZES+1; i++) { extent_heap_new(&arena->extents_cached[i]); extent_heap_new(&arena->extents_retained[i]); } diff --git a/src/extent.c b/src/extent.c index ad78c879..a802ad90 100644 --- a/src/extent.c +++ b/src/extent.c @@ -45,7 +45,7 @@ static size_t highpages; */ static void extent_record(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES], + extent_hooks_t **r_extent_hooks, extent_heap_t extent_heaps[NPSIZES+1], bool cache, extent_t *extent); /******************************************************************************/ @@ -190,11 +190,11 @@ extent_ad_comp(const extent_t *a, const extent_t *b) ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_ad_comp) static void -extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES], +extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], extent_t *extent) { size_t psz = extent_size_quantize_floor(extent_size_get(extent)); - pszind_t pind = psz2ind_clamp(psz); + pszind_t pind = psz2ind(psz); malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); @@ -202,11 +202,11 @@ extent_heaps_insert(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES], } static void -extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES], +extent_heaps_remove(tsdn_t *tsdn, extent_heap_t extent_heaps[NPSIZES+1], extent_t *extent) { size_t psz = extent_size_quantize_floor(extent_size_get(extent)); - pszind_t pind = psz2ind_clamp(psz); + pszind_t pind = psz2ind(psz); malloc_mutex_assert_owner(tsdn, &extent_arena_get(extent)->extents_mtx); @@ -358,15 +358,14 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent) */ static extent_t * extent_first_best_fit(tsdn_t *tsdn, arena_t *arena, - extent_heap_t extent_heaps[NPSIZES], size_t size) + extent_heap_t extent_heaps[NPSIZES+1], size_t size) { pszind_t pind, i; malloc_mutex_assert_owner(tsdn, &arena->extents_mtx); pind = psz2ind(extent_size_quantize_ceil(size)); - assert(pind < NPSIZES); - for (i = pind; i < NPSIZES; i++) { + for (i = pind; i < NPSIZES+1; i++) { extent_t *extent = extent_heap_first(&extent_heaps[i]); if (extent != NULL) return (extent); @@ -393,7 +392,7 @@ extent_leak(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, static extent_t * extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_heap_t extent_heaps[NPSIZES], bool locked, bool cache, + extent_heap_t extent_heaps[NPSIZES+1], bool locked, bool cache, void *new_addr, size_t usize, size_t pad, size_t alignment, bool *zero, bool *commit, bool slab) { @@ -758,7 +757,7 @@ extent_can_coalesce(const extent_t *a, const extent_t *b) static void extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *a, extent_t *b, - extent_heap_t extent_heaps[NPSIZES], bool cache) + extent_heap_t extent_heaps[NPSIZES+1], bool cache) { if (!extent_can_coalesce(a, b)) @@ -786,7 +785,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, static void extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, - extent_heap_t extent_heaps[NPSIZES], bool cache, extent_t *extent) + extent_heap_t extent_heaps[NPSIZES+1], bool cache, extent_t *extent) { extent_t *prev, *next; rtree_ctx_t rtree_ctx_fallback; diff --git a/src/jemalloc.c b/src/jemalloc.c index 86030172..dc0add49 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -83,7 +83,7 @@ enum { static uint8_t malloc_slow_flags; JEMALLOC_ALIGNED(CACHELINE) -const size_t pind2sz_tab[NPSIZES] = { +const size_t pind2sz_tab[NPSIZES+1] = { #define PSZ_yes(lg_grp, ndelta, lg_delta) \ (((ZU(1)<