diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index a6f6f7c3..ddfa0046 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -22,8 +22,6 @@ extern size_t opt_lg_chunk; extern const char *opt_dss; -extern rtree_t chunks_rtree; - extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ extern size_t chunk_npages; @@ -67,19 +65,6 @@ void chunk_postfork_child(tsdn_t *tsdn); /******************************************************************************/ #ifdef JEMALLOC_H_INLINES -#ifndef JEMALLOC_ENABLE_INLINE -extent_t *chunk_lookup(tsdn_t *tsdn, const void *chunk, bool dependent); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_CHUNK_C_)) -JEMALLOC_INLINE extent_t * -chunk_lookup(tsdn_t *tsdn, const void *ptr, bool dependent) -{ - - return (rtree_read(tsdn, &chunks_rtree, (uintptr_t)ptr, dependent)); -} -#endif - #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index b590ae0f..8552f701 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -87,6 +87,8 @@ typedef ph(extent_t) extent_heap_t; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS +extern rtree_t extents_rtree; + extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent); @@ -101,11 +103,14 @@ size_t extent_size_quantize_ceil(size_t size); ph_proto(, extent_heap_, extent_heap_t, extent_t) +bool extent_boot(void); + #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +extent_t *extent_lookup(tsdn_t *tsdn, const void *chunk, bool dependent); arena_t *extent_arena_get(const extent_t *extent); void *extent_base_get(const extent_t *extent); void *extent_addr_get(const extent_t *extent); @@ -140,6 +145,13 @@ void extent_ring_remove(extent_t *extent); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) +JEMALLOC_INLINE extent_t * +extent_lookup(tsdn_t *tsdn, const void *ptr, bool dependent) +{ + + return (rtree_read(tsdn, &extents_rtree, (uintptr_t)ptr, dependent)); +} + JEMALLOC_INLINE arena_t * extent_arena_get(const extent_t *extent) { diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 58a18ae5..fb3991bc 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -510,9 +510,9 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/witness.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" -#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/pages.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/large.h" @@ -929,7 +929,7 @@ JEMALLOC_ALWAYS_INLINE extent_t * iealloc(tsdn_t *tsdn, const void *ptr) { - return (chunk_lookup(tsdn, ptr, true)); + return (extent_lookup(tsdn, ptr, true)); } #endif @@ -1062,7 +1062,7 @@ ivsalloc(tsdn_t *tsdn, const void *ptr) extent_t *extent; /* Return 0 if ptr is not within a chunk managed by jemalloc. */ - extent = chunk_lookup(tsdn, ptr, false); + extent = extent_lookup(tsdn, ptr, false); if (extent == NULL) return (0); assert(extent_active_get(extent)); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index b5fd4c0c..92f91e44 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -132,7 +132,6 @@ chunk_dss_prec_get chunk_dss_prec_set chunk_dss_prefork chunk_in_dss -chunk_lookup chunk_merge_wrapper chunk_npages chunk_postfork_child @@ -140,7 +139,6 @@ chunk_postfork_parent chunk_prefork chunk_purge_wrapper chunk_split_wrapper -chunks_rtree chunksize chunksize_mask ckh_count @@ -173,6 +171,7 @@ extent_arena_get extent_arena_set extent_base_get extent_before_get +extent_boot extent_committed_get extent_committed_set extent_dalloc @@ -187,6 +186,7 @@ extent_hooks_get extent_hooks_set extent_init extent_last_get +extent_lookup extent_past_get extent_prof_tctx_get extent_prof_tctx_set @@ -205,6 +205,7 @@ extent_usize_get extent_usize_set extent_zeroed_get extent_zeroed_set +extents_rtree ffs_llu ffs_lu ffs_u diff --git a/src/chunk.c b/src/chunk.c index 7f80bb42..3ce6e015 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -11,8 +11,6 @@ size_t opt_lg_chunk = 0; static size_t curchunks; static size_t highchunks; -rtree_t chunks_rtree; - /* Various chunk-related settings. */ size_t chunksize; size_t chunksize_mask; /* (chunksize - 1). */ @@ -161,14 +159,14 @@ extent_rtree_acquire(tsdn_t *tsdn, const extent_t *extent, bool dependent, bool init_missing, rtree_elm_t **r_elm_a, rtree_elm_t **r_elm_b) { - *r_elm_a = rtree_elm_acquire(tsdn, &chunks_rtree, + *r_elm_a = rtree_elm_acquire(tsdn, &extents_rtree, (uintptr_t)extent_base_get(extent), dependent, init_missing); if (!dependent && *r_elm_a == NULL) return (true); assert(*r_elm_a != NULL); if (extent_size_get(extent) > PAGE) { - *r_elm_b = rtree_elm_acquire(tsdn, &chunks_rtree, + *r_elm_b = rtree_elm_acquire(tsdn, &extents_rtree, (uintptr_t)extent_last_get(extent), dependent, init_missing); if (!dependent && *r_elm_b == NULL) @@ -185,18 +183,18 @@ extent_rtree_write_acquired(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b, const extent_t *extent) { - rtree_elm_write_acquired(tsdn, &chunks_rtree, elm_a, extent); + rtree_elm_write_acquired(tsdn, &extents_rtree, elm_a, extent); if (elm_b != NULL) - rtree_elm_write_acquired(tsdn, &chunks_rtree, elm_b, extent); + rtree_elm_write_acquired(tsdn, &extents_rtree, elm_b, extent); } static void extent_rtree_release(tsdn_t *tsdn, rtree_elm_t *elm_a, rtree_elm_t *elm_b) { - rtree_elm_release(tsdn, &chunks_rtree, elm_a); + rtree_elm_release(tsdn, &extents_rtree, elm_a); if (elm_b != NULL) - rtree_elm_release(tsdn, &chunks_rtree, elm_b); + rtree_elm_release(tsdn, &extents_rtree, elm_b); } static void @@ -207,7 +205,7 @@ chunk_interior_register(tsdn_t *tsdn, const extent_t *extent) assert(extent_slab_get(extent)); for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) { - rtree_write(tsdn, &chunks_rtree, + rtree_write(tsdn, &extents_rtree, (uintptr_t)extent_base_get(extent) + (uintptr_t)(i << LG_PAGE), extent); } @@ -252,7 +250,7 @@ chunk_interior_deregister(tsdn_t *tsdn, const extent_t *extent) assert(extent_slab_get(extent)); for (i = 1; i < (extent_size_get(extent) >> LG_PAGE) - 1; i++) { - rtree_clear(tsdn, &chunks_rtree, + rtree_clear(tsdn, &extents_rtree, (uintptr_t)extent_base_get(extent) + (uintptr_t)(i << LG_PAGE)); } @@ -335,15 +333,15 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, if (new_addr != NULL) { rtree_elm_t *elm; - elm = rtree_elm_acquire(tsdn, &chunks_rtree, + elm = rtree_elm_acquire(tsdn, &extents_rtree, (uintptr_t)new_addr, false, false); if (elm != NULL) { - extent = rtree_elm_read_acquired(tsdn, &chunks_rtree, + extent = rtree_elm_read_acquired(tsdn, &extents_rtree, elm); if (extent != NULL && (extent_active_get(extent) || extent_retained_get(extent) == cache)) extent = NULL; - rtree_elm_release(tsdn, &chunks_rtree, elm); + rtree_elm_release(tsdn, &extents_rtree, elm); } else extent = NULL; } else @@ -651,12 +649,12 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_slab_set(extent, false); } - assert(chunk_lookup(tsdn, extent_base_get(extent), true) == extent); + assert(extent_lookup(tsdn, extent_base_get(extent), true) == extent); extent_heaps_insert(extent_heaps, extent); arena_chunk_cache_maybe_insert(arena, extent, cache); /* Try to coalesce forward. */ - next = rtree_read(tsdn, &chunks_rtree, + next = rtree_read(tsdn, &extents_rtree, (uintptr_t)extent_past_get(extent), false); if (next != NULL) { chunk_try_coalesce(tsdn, arena, extent_hooks, extent, next, @@ -664,7 +662,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, } /* Try to coalesce backward. */ - prev = rtree_read(tsdn, &chunks_rtree, + prev = rtree_read(tsdn, &extents_rtree, (uintptr_t)extent_before_get(extent), false); if (prev != NULL) { chunk_try_coalesce(tsdn, arena, extent_hooks, prev, extent, @@ -907,12 +905,12 @@ chunk_merge_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t *extent_hooks, extent_rtree_acquire(tsdn, b, true, false, &b_elm_a, &b_elm_b); if (a_elm_b != NULL) { - rtree_elm_write_acquired(tsdn, &chunks_rtree, a_elm_b, NULL); - rtree_elm_release(tsdn, &chunks_rtree, a_elm_b); + rtree_elm_write_acquired(tsdn, &extents_rtree, a_elm_b, NULL); + rtree_elm_release(tsdn, &extents_rtree, a_elm_b); } if (b_elm_b != NULL) { - rtree_elm_write_acquired(tsdn, &chunks_rtree, b_elm_a, NULL); - rtree_elm_release(tsdn, &chunks_rtree, b_elm_a); + rtree_elm_write_acquired(tsdn, &extents_rtree, b_elm_a, NULL); + rtree_elm_release(tsdn, &extents_rtree, b_elm_a); } else b_elm_b = b_elm_a; @@ -963,9 +961,6 @@ chunk_boot(void) if (have_dss && chunk_dss_boot()) return (true); - if (rtree_new(&chunks_rtree, (unsigned)((ZU(1) << (LG_SIZEOF_PTR+3)) - - LG_PAGE))) - return (true); return (false); } diff --git a/src/extent.c b/src/extent.c index 2f929a83..3e62e3bc 100644 --- a/src/extent.c +++ b/src/extent.c @@ -1,6 +1,11 @@ #define JEMALLOC_EXTENT_C_ #include "jemalloc/internal/jemalloc_internal.h" +/******************************************************************************/ +/* Data. */ + +rtree_t extents_rtree; + /******************************************************************************/ extent_t * @@ -112,3 +117,14 @@ extent_ad_comp(const extent_t *a, const extent_t *b) /* Generate pairing heap functions. */ ph_gen(, extent_heap_, extent_heap_t, extent_t, ph_link, extent_ad_comp) + +bool +extent_boot(void) +{ + + if (rtree_new(&extents_rtree, (unsigned)((ZU(1) << (LG_SIZEOF_PTR+3)) - + LG_PAGE))) + return (true); + + return (false); +} diff --git a/src/jemalloc.c b/src/jemalloc.c index a9bba12b..03e61df6 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1232,6 +1232,8 @@ malloc_init_hard_a0_locked() return (true); if (chunk_boot()) return (true); + if (extent_boot()) + return (true); if (ctl_boot()) return (true); if (config_prof) diff --git a/test/unit/arena_reset.c b/test/unit/arena_reset.c index a9476b89..61caf3c5 100644 --- a/test/unit/arena_reset.c +++ b/test/unit/arena_reset.c @@ -70,7 +70,7 @@ vsalloc(tsdn_t *tsdn, const void *ptr) { extent_t *extent; - extent = chunk_lookup(tsdn, ptr, false); + extent = extent_lookup(tsdn, ptr, false); if (extent == NULL) return (0); if (!extent_active_get(extent))