From 2fe5108263d013b07572f5aa597ba6ace86ed342 Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 3 Dec 2019 14:03:38 -0800 Subject: [PATCH] Extent -> Ehooks: Move merge hook. --- include/jemalloc/internal/ehooks.h | 19 ++++++--- include/jemalloc/internal/extent_externs.h | 1 + src/ehooks.c | 26 ++++++++++++ src/extent.c | 48 +++------------------- 4 files changed, 46 insertions(+), 48 deletions(-) diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index e84222f6..48d13fc6 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -42,6 +42,9 @@ bool ehooks_default_purge_forced(extent_hooks_t *extent_hooks, void *addr, bool ehooks_default_split_impl(); bool ehooks_default_split(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind); +bool ehooks_default_merge_impl(void *addr_a, void *addr_b); +bool ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + void *addr_b, size_t size_b, bool committed, unsigned arena_ind); static inline void ehooks_pre_reentrancy(tsdn_t *tsdn) { @@ -238,14 +241,20 @@ ehooks_split(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, } static inline bool -ehooks_merge(ehooks_t *ehooks, void *addr_a, size_t size_a, void *addr_b, - size_t size_b, bool committed, unsigned arena_ind) { +ehooks_merge(tsdn_t *tsdn, ehooks_t *ehooks, void *addr_a, size_t size_a, + void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); - if (extent_hooks->merge == NULL) { + if (extent_hooks == &extent_hooks_default) { + return ehooks_default_merge_impl(addr_a, addr_b); + } else if (extent_hooks->merge == NULL) { return true; + } else { + ehooks_pre_reentrancy(tsdn); + bool err = extent_hooks->merge(extent_hooks, addr_a, size_a, + addr_b, size_b, committed, arena_ind); + ehooks_post_reentrancy(tsdn); + return err; } - return extent_hooks->merge(extent_hooks, addr_a, size_a, addr_b, size_b, - committed, arena_ind); } #endif /* JEMALLOC_INTERNAL_EHOOKS_H */ diff --git a/include/jemalloc/internal/extent_externs.h b/include/jemalloc/internal/extent_externs.h index 26828ba5..4e3803c6 100644 --- a/include/jemalloc/internal/extent_externs.h +++ b/include/jemalloc/internal/extent_externs.h @@ -46,6 +46,7 @@ extent_t *extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, size_t size_b, szind_t szind_b, bool slab_b); bool extent_merge_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a, extent_t *b); +bool extent_head_no_merge(extent_t *a, extent_t *b); bool extent_boot(void); diff --git a/src/ehooks.c b/src/ehooks.c index 8bd95500..bb328546 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -182,3 +182,29 @@ ehooks_default_split(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t size_a, size_t size_b, bool committed, unsigned arena_ind) { return ehooks_default_split_impl(); } + +bool +ehooks_default_merge_impl(void *addr_a, void *addr_b) { + if (!maps_coalesce && !opt_retain) { + return true; + } + if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { + return true; + } + + return false; +} + +bool +ehooks_default_merge(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, + void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { + if (!maps_coalesce) { + tsdn_t *tsdn = tsdn_fetch(); + extent_t *a = iealloc(tsdn, addr_a); + extent_t *b = iealloc(tsdn, addr_b); + if (extent_head_no_merge(a, b)) { + return true; + } + } + return ehooks_default_merge_impl(addr_a, addr_b); +} diff --git a/src/extent.c b/src/extent.c index 521c0b96..3e78e965 100644 --- a/src/extent.c +++ b/src/extent.c @@ -31,9 +31,6 @@ static extent_t *extent_split_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t size_a, szind_t szind_a, bool slab_a, size_t size_b, szind_t szind_b, bool slab_b, bool growing_retained); -static bool extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, - size_t size_a, void *addr_b, size_t size_b, bool committed, - unsigned arena_ind); static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a, extent_t *b, bool growing_retained); @@ -54,7 +51,7 @@ const extent_hooks_t extent_hooks_default = { NULL, #endif ehooks_default_split, - extent_merge_default + ehooks_default_merge }; /* Used exclusively for gdump triggering. */ @@ -1576,23 +1573,11 @@ extent_split_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, slab_a, size_b, szind_b, slab_b, false); } -static bool -extent_merge_default_impl(void *addr_a, void *addr_b) { - if (!maps_coalesce && !opt_retain) { - return true; - } - if (have_dss && !extent_dss_mergeable(addr_a, addr_b)) { - return true; - } - - return false; -} - /* * Returns true if the given extents can't be merged because of their head bit * settings. Assumes the second extent has the higher address. */ -static bool +bool extent_head_no_merge(extent_t *a, extent_t *b) { assert(extent_base_get(a) < extent_base_get(b)); /* @@ -1620,20 +1605,6 @@ extent_head_no_merge(extent_t *a, extent_t *b) { return false; } -static bool -extent_merge_default(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, - void *addr_b, size_t size_b, bool committed, unsigned arena_ind) { - if (!maps_coalesce) { - tsdn_t *tsdn = tsdn_fetch(); - extent_t *a = iealloc(tsdn, addr_a); - extent_t *b = iealloc(tsdn, addr_b); - if (extent_head_no_merge(a, b)) { - return true; - } - } - return extent_merge_default_impl(addr_a, addr_b); -} - static bool extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a, extent_t *b, bool growing_retained) { @@ -1645,18 +1616,9 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *a, return true; } - bool err; - if (ehooks_are_default(ehooks)) { - /* Call directly to propagate tsdn. */ - err = extent_merge_default_impl(extent_base_get(a), - extent_base_get(b)); - } else { - extent_hook_pre_reentrancy(tsdn, arena); - err = ehooks_merge(ehooks, extent_base_get(a), - extent_size_get(a), extent_base_get(b), extent_size_get(b), - extent_committed_get(a), arena_ind_get(arena)); - extent_hook_post_reentrancy(tsdn); - } + bool err = ehooks_merge(tsdn, ehooks, extent_base_get(a), + extent_size_get(a), extent_base_get(b), extent_size_get(b), + extent_committed_get(a), arena_ind_get(arena)); if (err) { return true;