From d78fe241acb79ab4b0b7cb5b48d07be8582fc60a Mon Sep 17 00:00:00 2001 From: David Goldblatt Date: Tue, 3 Dec 2019 12:25:00 -0800 Subject: [PATCH] Extent -> Ehooks: Move commit and decommit hooks. --- include/jemalloc/internal/ehooks.h | 38 ++++++++++++++++++++-------- src/base.c | 2 +- src/ehooks.c | 23 +++++++++++++++++ src/extent.c | 40 +++--------------------------- 4 files changed, 56 insertions(+), 47 deletions(-) diff --git a/include/jemalloc/internal/ehooks.h b/include/jemalloc/internal/ehooks.h index 07094d91..e9bdca37 100644 --- a/include/jemalloc/internal/ehooks.h +++ b/include/jemalloc/internal/ehooks.h @@ -23,6 +23,12 @@ bool ehooks_default_dalloc(extent_hooks_t *extent_hooks, void *addr, void ehooks_default_destroy_impl(void *addr, size_t size); void ehooks_default_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind); +bool ehooks_default_commit_impl(void *addr, size_t offset, size_t length); +bool ehooks_default_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind); +bool ehooks_default_decommit_impl(void *addr, size_t offset, size_t length); +bool ehooks_default_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind); static inline void ehooks_pre_reentrancy(tsdn_t *tsdn) { @@ -128,25 +134,37 @@ ehooks_destroy(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, } static inline bool -ehooks_commit(ehooks_t *ehooks, void *addr, size_t size, size_t offset, - size_t length, unsigned arena_ind) { +ehooks_commit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); - if (extent_hooks->commit == NULL) { + if (extent_hooks == &extent_hooks_default) { + return ehooks_default_commit_impl(addr, offset, length); + } else if (extent_hooks->commit == NULL) { return true; + } else { + ehooks_pre_reentrancy(tsdn); + bool err = extent_hooks->commit(extent_hooks, addr, size, + offset, length, arena_ind); + ehooks_post_reentrancy(tsdn); + return err; } - return extent_hooks->commit(extent_hooks, addr, size, offset, length, - arena_ind); } static inline bool -ehooks_decommit(ehooks_t *ehooks, void *addr, size_t size, size_t offset, - size_t length, unsigned arena_ind) { +ehooks_decommit(tsdn_t *tsdn, ehooks_t *ehooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { extent_hooks_t *extent_hooks = ehooks_get_extent_hooks_ptr(ehooks); - if (extent_hooks->decommit == NULL) { + if (extent_hooks == &extent_hooks_default) { + return ehooks_default_decommit_impl(addr, offset, length); + } else if (extent_hooks->decommit == NULL) { return true; + } else { + ehooks_pre_reentrancy(tsdn); + bool err = extent_hooks->decommit(extent_hooks, addr, size, + offset, length, arena_ind); + ehooks_post_reentrancy(tsdn); + return err; } - return extent_hooks->decommit(extent_hooks, addr, size, offset, length, - arena_ind); } static inline bool diff --git a/src/base.c b/src/base.c index 52699c50..6b88b238 100644 --- a/src/base.c +++ b/src/base.c @@ -84,7 +84,7 @@ base_unmap(tsdn_t *tsdn, ehooks_t *ehooks, unsigned ind, void *addr, if (!ehooks_dalloc(tsdn, ehooks, addr, size, true, ind)) { goto label_post_reentrancy; } - if (!ehooks_decommit(ehooks, addr, size, 0, size, ind)) { + if (!ehooks_decommit(tsdn, ehooks, addr, size, 0, size, ind)) { goto label_post_reentrancy; } if (!ehooks_purge_forced(ehooks, addr, size, 0, size, ind)) { diff --git a/src/ehooks.c b/src/ehooks.c index ad6fd24e..cb02377c 100644 --- a/src/ehooks.c +++ b/src/ehooks.c @@ -103,3 +103,26 @@ ehooks_default_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, ehooks_default_destroy_impl(addr, size); } +bool +ehooks_default_commit_impl(void *addr, size_t offset, size_t length) { + return pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), + length); +} + +bool +ehooks_default_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { + return ehooks_default_commit_impl(addr, offset, length); +} + +bool +ehooks_default_decommit_impl(void *addr, size_t offset, size_t length) { + return pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), + length); +} + +bool +ehooks_default_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, + size_t offset, size_t length, unsigned arena_ind) { + return ehooks_default_decommit_impl(addr, offset, length); +} diff --git a/src/extent.c b/src/extent.c index 271fe4a3..3eb4961b 100644 --- a/src/extent.c +++ b/src/extent.c @@ -19,12 +19,8 @@ mutex_pool_t extent_mutex_pool; size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; -static bool extent_commit_default(extent_hooks_t *extent_hooks, void *addr, - size_t size, size_t offset, size_t length, unsigned arena_ind); static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained); -static bool extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, - size_t size, size_t offset, size_t length, unsigned arena_ind); #ifdef PAGES_CAN_PURGE_LAZY static bool extent_purge_lazy_default(extent_hooks_t *extent_hooks, void *addr, size_t size, size_t offset, size_t length, unsigned arena_ind); @@ -56,8 +52,8 @@ const extent_hooks_t extent_hooks_default = { ehooks_default_alloc, ehooks_default_dalloc, ehooks_default_destroy, - extent_commit_default, - extent_decommit_default + ehooks_default_commit, + ehooks_default_decommit #ifdef PAGES_CAN_PURGE_LAZY , extent_purge_lazy_default @@ -1440,27 +1436,13 @@ extent_destroy_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_dalloc(tsdn, arena, extent); } -static bool -extent_commit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, - size_t offset, size_t length, unsigned arena_ind) { - return pages_commit((void *)((uintptr_t)addr + (uintptr_t)offset), - length); -} - static bool extent_commit_impl(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length, bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); - - if (!ehooks_are_default(ehooks)) { - extent_hook_pre_reentrancy(tsdn, arena); - } - bool err = ehooks_commit(ehooks, extent_base_get(extent), + bool err = ehooks_commit(tsdn, ehooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena)); - if (!ehooks_are_default(ehooks)) { - extent_hook_post_reentrancy(tsdn); - } extent_committed_set(extent, extent_committed_get(extent) || !err); return err; } @@ -1473,27 +1455,13 @@ extent_commit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, false); } -static bool -extent_decommit_default(extent_hooks_t *extent_hooks, void *addr, size_t size, - size_t offset, size_t length, unsigned arena_ind) { - return pages_decommit((void *)((uintptr_t)addr + (uintptr_t)offset), - length); -} - bool extent_decommit_wrapper(tsdn_t *tsdn, arena_t *arena, ehooks_t *ehooks, extent_t *extent, size_t offset, size_t length) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - - if (!ehooks_are_default(ehooks)) { - extent_hook_pre_reentrancy(tsdn, arena); - } - bool err = ehooks_decommit(ehooks, extent_base_get(extent), + bool err = ehooks_decommit(tsdn, ehooks, extent_base_get(extent), extent_size_get(extent), offset, length, arena_ind_get(arena)); - if (!ehooks_are_default(ehooks)) { - extent_hook_post_reentrancy(tsdn); - } extent_committed_set(extent, extent_committed_get(extent) && err); return err; }