diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index acc67f00..33f5932c 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -18,6 +18,9 @@ struct extent_s { /* Total region size. */ size_t e_size; + /* True if extent is active (in use). */ + bool e_active; + /* * The zeroed flag is used by chunk recycling code to track whether * memory is zero-filled. @@ -73,6 +76,7 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_t) arena_t *extent_arena_get(const extent_t *extent); void *extent_addr_get(const extent_t *extent); size_t extent_size_get(const extent_t *extent); +bool extent_active_get(const extent_t *extent); bool extent_zeroed_get(const extent_t *extent); bool extent_committed_get(const extent_t *extent); bool extent_achunk_get(const extent_t *extent); @@ -80,13 +84,13 @@ prof_tctx_t *extent_prof_tctx_get(const extent_t *extent); void extent_arena_set(extent_t *extent, arena_t *arena); void extent_addr_set(extent_t *extent, void *addr); void extent_size_set(extent_t *extent, size_t size); +void extent_active_set(extent_t *extent, bool active); void extent_zeroed_set(extent_t *extent, bool zeroed); void extent_committed_set(extent_t *extent, bool committed); void extent_achunk_set(extent_t *extent, bool achunk); void extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx); void extent_init(extent_t *extent, arena_t *arena, void *addr, - size_t size, bool zeroed, bool committed); -void extent_dirty_linkage_init(extent_t *extent); + size_t size, bool active, bool zeroed, bool committed); void extent_dirty_insert(extent_t *extent, arena_runs_dirty_link_t *runs_dirty, extent_t *chunks_dirty); void extent_dirty_remove(extent_t *extent); @@ -114,6 +118,13 @@ extent_size_get(const extent_t *extent) return (extent->e_size); } +JEMALLOC_INLINE bool +extent_active_get(const extent_t *extent) +{ + + return (extent->e_active); +} + JEMALLOC_INLINE bool extent_zeroed_get(const extent_t *extent) { @@ -164,6 +175,13 @@ extent_size_set(extent_t *extent, size_t size) extent->e_size = size; } +JEMALLOC_INLINE void +extent_active_set(extent_t *extent, bool active) +{ + + extent->e_active = active; +} + JEMALLOC_INLINE void extent_zeroed_set(extent_t *extent, bool zeroed) { @@ -194,23 +212,18 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) JEMALLOC_INLINE void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, - bool zeroed, bool committed) + bool active, bool zeroed, bool committed) { extent_arena_set(extent, arena); extent_addr_set(extent, addr); extent_size_set(extent, size); + extent_active_set(extent, active); extent_zeroed_set(extent, zeroed); extent_committed_set(extent, committed); extent_achunk_set(extent, false); if (config_prof) extent_prof_tctx_set(extent, NULL); -} - -JEMALLOC_INLINE void -extent_dirty_linkage_init(extent_t *extent) -{ - qr_new(&extent->rd, rd_link); qr_new(extent, cc_link); } diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index c7d97371..a7f07818 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -1093,6 +1093,7 @@ ivsalloc(tsdn_t *tsdn, const void *ptr, bool demote) extent = chunk_lookup(tsdn, ptr, false); if (extent == NULL) return (0); + assert(extent_active_get(extent)); /* Only arena chunks should be looked up via interior pointers. */ assert(extent_addr_get(extent) == ptr || extent_achunk_get(extent)); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 102f01c0..19402462 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -207,6 +207,8 @@ decay_ticker_get dss_prec_names extent_achunk_get extent_achunk_set +extent_active_get +extent_active_set extent_addr_get extent_addr_set extent_arena_get @@ -214,7 +216,6 @@ extent_arena_set extent_committed_get extent_committed_set extent_dirty_insert -extent_dirty_linkage_init extent_dirty_remove extent_init extent_prof_tctx_get diff --git a/src/arena.c b/src/arena.c index 8a93fca0..5a4605d0 100644 --- a/src/arena.c +++ b/src/arena.c @@ -225,7 +225,6 @@ arena_chunk_cache_maybe_insert(arena_t *arena, extent_t *extent, bool cache) { if (cache) { - extent_dirty_linkage_init(extent); extent_dirty_insert(extent, &arena->runs_dirty, &arena->chunks_cache); arena->ndirty += arena_chunk_dirty_npages(extent); @@ -526,7 +525,7 @@ arena_chunk_register(tsdn_t *tsdn, arena_t *arena, arena_chunk_t *chunk, * runs is tracked individually, and upon chunk deallocation the entire * chunk is in a consistent commit state. */ - extent_init(&chunk->extent, arena, chunk, chunksize, zero, true); + extent_init(&chunk->extent, arena, chunk, chunksize, true, zero, true); extent_achunk_set(&chunk->extent, true); return (chunk_register(tsdn, chunk, &chunk->extent)); } @@ -1723,7 +1722,8 @@ arena_purge_to_limit(tsdn_t *tsdn, arena_t *arena, size_t ndirty_limit) arena->lg_dirty_mult) < arena->ndirty || ndirty_limit == 0); qr_new(&purge_runs_sentinel, rd_link); - extent_dirty_linkage_init(&purge_chunks_sentinel); + extent_init(&purge_chunks_sentinel, arena, NULL, 0, false, false, + false); npurge = arena_stash_dirty(tsdn, arena, &chunk_hooks, ndirty_limit, &purge_runs_sentinel, &purge_chunks_sentinel); diff --git a/src/base.c b/src/base.c index a9ab279e..0176fb80 100644 --- a/src/base.c +++ b/src/base.c @@ -66,7 +66,7 @@ base_chunk_alloc(tsdn_t *tsdn, size_t minsize) base_resident += PAGE_CEILING(nsize); } } - extent_init(extent, NULL, addr, csize, true, true); + extent_init(extent, NULL, addr, csize, true, true, true); return (extent); } @@ -90,7 +90,7 @@ base_alloc(tsdn_t *tsdn, size_t size) csize = CACHELINE_CEILING(size); usize = s2u(csize); - extent_init(&key, NULL, NULL, usize, false, false); + extent_init(&key, NULL, NULL, usize, false, false, false); malloc_mutex_lock(tsdn, &base_mtx); extent = extent_tree_szad_nsearch(&base_avail_szad, &key); if (extent != NULL) { diff --git a/src/chunk.c b/src/chunk.c index beef41fe..055b4fcb 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -239,7 +239,7 @@ chunk_first_best_fit(arena_t *arena, extent_tree_t *chunks_szad, assert(size == CHUNK_CEILING(size)); - extent_init(&key, arena, NULL, size, false, false); + extent_init(&key, arena, NULL, size, false, false, false); return (extent_tree_szad_nsearch(chunks_szad, &key)); } @@ -270,7 +270,8 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks); if (new_addr != NULL) { extent_t key; - extent_init(&key, arena, new_addr, alloc_size, false, false); + extent_init(&key, arena, new_addr, alloc_size, false, false, + false); extent = extent_tree_ad_search(chunks_ad, &key); } else { extent = chunk_first_best_fit(arena, chunks_szad, chunks_ad, @@ -336,7 +337,7 @@ chunk_recycle(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, } } extent_init(extent, arena, (void *)((uintptr_t)(ret) + size), - trailsize, zeroed, committed); + trailsize, false, zeroed, committed); extent_tree_szad_insert(chunks_szad, extent); extent_tree_ad_insert(chunks_ad, extent); arena_chunk_cache_maybe_insert(arena, extent, cache); @@ -534,7 +535,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, malloc_mutex_lock(tsdn, &arena->chunks_mtx); chunk_hooks_assure_initialized_locked(tsdn, arena, chunk_hooks); extent_init(&key, arena, (void *)((uintptr_t)chunk + size), 0, false, - false); + false, false); extent = extent_tree_ad_nsearch(chunks_ad, &key); /* Try to coalesce forward. */ if (extent != NULL && extent_addr_get(extent) == extent_addr_get(&key) @@ -570,7 +571,7 @@ chunk_record(tsdn_t *tsdn, arena_t *arena, chunk_hooks_t *chunk_hooks, } goto label_return; } - extent_init(extent, arena, chunk, size, !unzeroed, + extent_init(extent, arena, chunk, size, false, !unzeroed, committed); extent_tree_ad_insert(chunks_ad, extent); extent_tree_szad_insert(chunks_szad, extent); diff --git a/src/huge.c b/src/huge.c index 0b91c369..b6ad4ba6 100644 --- a/src/huge.c +++ b/src/huge.c @@ -50,7 +50,7 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, return (NULL); } - extent_init(extent, arena, ret, usize, is_zeroed, true); + extent_init(extent, arena, ret, usize, true, is_zeroed, true); if (chunk_register(tsdn, ret, extent)) { arena_chunk_dalloc_huge(tsdn, arena, ret, usize);