romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-04-28 05:42:17 +03:00
Code Issues Projects Releases Packages Wiki Activity

Use rtree tracked states to protect edata outside of ecache locks.

Browse source 
This avoids the addr-based mutexes (i.e. the mutex_pool), and instead relies on
the metadata tracked in rtree leaf: the head state and extent_state.  Before
trying to access the neighbor edata (e.g. for coalescing), the states will be
verified first -- only neighbor edatas from the same arena and with the same
state will be accessed.
This commit is contained in:
Qi Wang 2021-03-04 14:33:40 -08:00 • committed by Qi Wang
parent 9ea235f8fe
commit 1784939688
7 changed files with 366 additions and 277 deletions
Download patch file Download diff file Expand all files Collapse all files

263
src/emap.c
View file

@ -3,15 +3,6 @@
#include "jemalloc/internal/emap.h"
/*
* Note: Ends without at semicolon, so that
* EMAP_DECLARE_RTREE_CTX;
* in uses will avoid empty-statement warnings.
*/
#define EMAP_DECLARE_RTREE_CTX \
rtree_ctx_t rtree_ctx_fallback; \
rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback)
enum emap_lock_result_e {
emap_lock_result_success,
emap_lock_result_failure,
@ -35,82 +26,186 @@ emap_init(emap_t *emap, base_t *base, bool zeroed) {
}
void
emap_lock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
assert(edata != NULL);
mutex_pool_lock(tsdn, &emap->mtx_pool, (uintptr_t)edata);
}
emap_update_edata_state(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
extent_state_t state) {
witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE);
void
emap_unlock_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
assert(edata != NULL);
mutex_pool_unlock(tsdn, &emap->mtx_pool, (uintptr_t)edata);
}
edata_state_set(edata, state);
void
emap_lock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_lock2(tsdn, &emap->mtx_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
void
emap_unlock_edata2(tsdn_t *tsdn, emap_t *emap, edata_t *edata1,
edata_t *edata2) {
assert(edata1 != NULL && edata2 != NULL);
mutex_pool_unlock2(tsdn, &emap->mtx_pool, (uintptr_t)edata1,
(uintptr_t)edata2);
}
static inline emap_lock_result_t
emap_try_lock_rtree_leaf_elm(tsdn_t *tsdn, emap_t *emap, rtree_leaf_elm_t *elm,
edata_t **result, bool inactive_only) {
edata_t *edata1 = rtree_leaf_elm_read(tsdn, &emap->rtree, elm,
/* dependent */ true).edata;
/* Slab implies active extents and should be skipped. */
if (edata1 == NULL || (inactive_only && rtree_leaf_elm_read(tsdn,
&emap->rtree, elm, /* dependent */ true).metadata.slab)) {
return emap_lock_result_no_extent;
}
/*
* It's possible that the extent changed out from under us, and with it
* the leaf->edata mapping. We have to recheck while holding the lock.
*/
emap_lock_edata(tsdn, emap, edata1);
edata_t *edata2 = rtree_leaf_elm_read(tsdn, &emap->rtree, elm,
/* dependent */ true).edata;
if (edata1 == edata2) {
*result = edata1;
return emap_lock_result_success;
} else {
emap_unlock_edata(tsdn, emap, edata1);
return emap_lock_result_failure;
}
}
/*
* Returns a pool-locked edata_t * if there's one associated with the given
* address, and NULL otherwise.
*/
edata_t *
emap_lock_edata_from_addr(tsdn_t *tsdn, emap_t *emap, void *addr,
bool inactive_only) {
EMAP_DECLARE_RTREE_CTX;
edata_t *ret = NULL;
rtree_leaf_elm_t *elm1 = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
rtree_ctx, (uintptr_t)edata_base_get(edata), /* dependent */ true,
/* init_missing */ false);
assert(elm1 != NULL);
rtree_leaf_elm_t *elm2 = edata_size_get(edata) == PAGE ? NULL :
rtree_leaf_elm_lookup(tsdn, &emap->rtree, rtree_ctx,
(uintptr_t)edata_last_get(edata), /* dependent */ true,
/* init_missing */ false);
rtree_leaf_elm_state_update(tsdn, &emap->rtree, elm1, elm2, state);
emap_assert_mapped(tsdn, emap, edata);
}
static inline bool
edata_neighbor_head_state_mergeable(bool edata_is_head,
bool neighbor_is_head, bool forward) {
/*
* Head states checking: disallow merging if the higher addr extent is a
* head extent. This helps preserve first-fit, and more importantly
* makes sure no merge across arenas.
*/
if (forward) {
if (neighbor_is_head) {
return false;
}
} else {
if (edata_is_head) {
return false;
}
}
return true;
}
static inline bool
edata_can_acquire_neighbor(edata_t *edata, rtree_contents_t contents,
extent_pai_t pai, extent_state_t expected_state, bool forward) {
edata_t *neighbor = contents.edata;
if (neighbor == NULL) {
return false;
}
/* It's not safe to access *neighbor yet; must verify states first. */
bool neighbor_is_head = contents.metadata.is_head;
if (!edata_neighbor_head_state_mergeable(edata_is_head_get(edata),
neighbor_is_head, forward)) {
return NULL;
}
extent_state_t neighbor_state = contents.metadata.state;
if (pai == EXTENT_PAI_PAC) {
if (neighbor_state != expected_state) {
return false;
}
/* From this point, it's safe to access *neighbor. */
if (edata_committed_get(edata) !=
edata_committed_get(neighbor)) {
/*
* Some platforms (e.g. Windows) require an explicit
* commit step (and writing to uncomitted memory is not
* allowed).
*/
return false;
}
} else {
if (neighbor_state == extent_state_active) {
return false;
}
/* From this point, it's safe to access *neighbor. */
}
assert(edata_pai_get(edata) == pai);
if (edata_pai_get(neighbor) != pai) {
return false;
}
if (opt_retain) {
assert(edata_arena_ind_get(edata) ==
edata_arena_ind_get(neighbor));
} else {
/*
* This isn't entirely safe with the presence of arena_reset /
* destroy, in which case the neighbor edata can be destoryed if
* it belongs to a manual arena. More on that later.
*/
if (edata_arena_ind_get(edata) !=
edata_arena_ind_get(neighbor)) {
return false;
}
}
return true;
}
/* Will be removed in the next commit. */
edata_t *
emap_try_acquire_edata(tsdn_t *tsdn, emap_t *emap, void *addr,
extent_state_t expected_state, bool allow_head_extent) {
EMAP_DECLARE_RTREE_CTX;
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
rtree_ctx, (uintptr_t)addr, false, false);
if (elm == NULL) {
return NULL;
}
emap_lock_result_t lock_result;
do {
lock_result = emap_try_lock_rtree_leaf_elm(tsdn, emap, elm,
&ret, inactive_only);
} while (lock_result == emap_lock_result_failure);
return ret;
rtree_contents_t contents = rtree_leaf_elm_read(tsdn, &emap->rtree, elm,
/* dependent */ true);
if (!allow_head_extent && contents.metadata.is_head) {
/* !allow_head_extent indicates the expanding path. */
return NULL;
}
edata_t *edata = contents.edata;
if (edata == NULL || contents.metadata.state != expected_state) {
return NULL;
}
assert(edata_state_get(edata) == expected_state);
emap_update_edata_state(tsdn, emap, edata, extent_state_updating);
return edata;
}
void
emap_release_edata(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
extent_state_t new_state) {
assert(emap_edata_in_transition(tsdn, emap, edata));
assert(emap_edata_is_acquired(tsdn, emap, edata));
emap_update_edata_state(tsdn, emap, edata, new_state);
}
edata_t *
emap_try_acquire_edata_neighbor(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
extent_pai_t pai, extent_state_t expected_state, bool forward) {
witness_assert_positive_depth_to_rank(tsdn_witness_tsdp_get(tsdn),
WITNESS_RANK_CORE);
assert(!edata_state_in_transition(expected_state));
assert(expected_state == extent_state_dirty ||
expected_state == extent_state_muzzy ||
expected_state == extent_state_retained);
void *neighbor_addr = forward ? edata_past_get(edata) :
edata_before_get(edata);
/*
* This is subtle; the rtree code asserts that its input pointer is
* non-NULL, and this is a useful thing to check. But it's possible
* that edata corresponds to an address of (void *)PAGE (in practice,
* this has only been observed on FreeBSD when address-space
* randomization is on, but it could in principle happen anywhere). In
* this case, edata_before_get(edata) is NULL, triggering the assert.
*/
if (neighbor_addr == NULL) {
return NULL;
}
EMAP_DECLARE_RTREE_CTX;
rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &emap->rtree,
rtree_ctx, (uintptr_t)neighbor_addr, /* dependent*/ false,
/* init_missing */ false);
if (elm == NULL) {
return NULL;
}
rtree_contents_t neighbor_contents = rtree_leaf_elm_read(tsdn,
&emap->rtree, elm, /* dependent */ true);
if (!edata_can_acquire_neighbor(edata, neighbor_contents, pai,
expected_state, forward)) {
return NULL;
}
/* From this point, the neighbor edata can be safely acquired. */
edata_t *neighbor = neighbor_contents.edata;
emap_update_edata_state(tsdn, emap, neighbor, extent_state_merging);
extent_assert_can_coalesce(edata, neighbor);
return neighbor;
}
static bool
@ -153,6 +248,7 @@ emap_rtree_write_acquired(tsdn_t *tsdn, emap_t *emap, rtree_leaf_elm_t *elm_a,
bool
emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
szind_t szind, bool slab) {
assert(edata_state_get(edata) == extent_state_active);
EMAP_DECLARE_RTREE_CTX;
rtree_leaf_elm_t *elm_a, *elm_b;
@ -161,6 +257,10 @@ emap_register_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
if (err) {
return true;
}
assert(rtree_leaf_elm_read(tsdn, &emap->rtree, elm_a,
/* dependent */ false).edata == NULL);
assert(rtree_leaf_elm_read(tsdn, &emap->rtree, elm_b,
/* dependent */ false).edata == NULL);
emap_rtree_write_acquired(tsdn, emap, elm_a, elm_b, edata, szind, slab);
return false;
}
@ -190,6 +290,15 @@ emap_register_interior(tsdn_t *tsdn, emap_t *emap, edata_t *edata,
void
emap_deregister_boundary(tsdn_t *tsdn, emap_t *emap, edata_t *edata) {
/*
* The edata must be either in an acquired state, or protected by state
* based locks.
*/
if (!emap_edata_is_acquired(tsdn, emap, edata)) {
witness_assert_positive_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE);
}
EMAP_DECLARE_RTREE_CTX;
rtree_leaf_elm_t *elm_a, *elm_b;

3
src/eset.c
View file

@ -78,7 +78,8 @@ eset_insert(eset_t *eset, edata_t *edata) {
void
eset_remove(eset_t *eset, edata_t *edata) {
assert(edata_state_get(edata) == eset->state);
assert(edata_state_get(edata) == eset->state ||
edata_state_in_transition(edata_state_get(edata)));
size_t size = edata_size_get(edata);
size_t psz = sz_psz_quantize_floor(size);

151
src/extent.c
View file

@ -64,12 +64,12 @@ extent_may_force_decay(pac_t *pac) {
static bool
extent_try_delayed_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
ecache_t *ecache, edata_t *edata) {
emap_edata_state_update(tsdn, pac->emap, edata, extent_state_active);
emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active);
bool coalesced;
edata = extent_try_coalesce(tsdn, pac, ehooks, ecache,
edata, &coalesced, false);
emap_edata_state_update(tsdn, pac->emap, edata, ecache->state);
emap_update_edata_state(tsdn, pac->emap, edata, ecache->state);
if (!coalesced) {
return true;
@ -183,7 +183,7 @@ ecache_evict(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
not_reached();
case extent_state_dirty:
case extent_state_muzzy:
emap_edata_state_update(tsdn, pac->emap, edata,
emap_update_edata_state(tsdn, pac->emap, edata,
extent_state_active);
break;
case extent_state_retained:
@ -230,7 +230,7 @@ extent_deactivate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache));
assert(edata_state_get(edata) == extent_state_active);
emap_edata_state_update(tsdn, pac->emap, edata, ecache->state);
emap_update_edata_state(tsdn, pac->emap, edata, ecache->state);
eset_insert(&ecache->eset, edata);
}
@ -245,10 +245,11 @@ static void
extent_activate_locked(tsdn_t *tsdn, pac_t *pac, ecache_t *ecache,
edata_t *edata) {
assert(edata_arena_ind_get(edata) == ecache_ind_get(ecache));
assert(edata_state_get(edata) == ecache->state);
assert(edata_state_get(edata) == ecache->state ||
edata_state_get(edata) == extent_state_updating);
eset_remove(&ecache->eset, edata);
emap_edata_state_update(tsdn, pac->emap, edata, extent_state_active);
emap_update_edata_state(tsdn, pac->emap, edata, extent_state_active);
}
static void
@ -290,20 +291,16 @@ extent_gdump_sub(tsdn_t *tsdn, const edata_t *edata) {
static bool
extent_register_impl(tsdn_t *tsdn, pac_t *pac, edata_t *edata, bool gdump_add) {
assert(edata_state_get(edata) == extent_state_active);
/*
* We need to hold the lock to protect against a concurrent coalesce
* operation that sees us in a partial state.
* No locking needed, as the edata must be in active state, which
* prevents other threads from accessing the edata.
*/
emap_lock_edata(tsdn, pac->emap, edata);
if (emap_register_boundary(tsdn, pac->emap, edata, SC_NSIZES,
/* slab */ false)) {
emap_unlock_edata(tsdn, pac->emap, edata);
return true;
}
emap_unlock_edata(tsdn, pac->emap, edata);
if (config_prof && gdump_add) {
extent_gdump_add(tsdn, edata);
}
@ -333,9 +330,7 @@ extent_reregister(tsdn_t *tsdn, pac_t *pac, edata_t *edata) {
static void
extent_deregister_impl(tsdn_t *tsdn, pac_t *pac, edata_t *edata,
bool gdump) {
emap_lock_edata(tsdn, pac->emap, edata);
emap_deregister_boundary(tsdn, pac->emap, edata);
emap_unlock_edata(tsdn, pac->emap, edata);
if (config_prof && gdump) {
extent_gdump_sub(tsdn, edata);
@ -383,22 +378,18 @@ extent_recycle_extract(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
malloc_mutex_lock(tsdn, &ecache->mtx);
edata_t *edata;
if (new_addr != NULL) {
edata = emap_lock_edata_from_addr(tsdn, pac->emap, new_addr,
false);
edata = emap_try_acquire_edata(tsdn, pac->emap, new_addr,
ecache->state, /* allow_head_extent*/ false);
if (edata != NULL) {
/*
* We might null-out edata to report an error, but we
* still need to unlock the associated mutex after.
*/
edata_t *unlock_edata = edata;
assert(edata_base_get(edata) == new_addr);
if (edata_arena_ind_get(edata) != ecache_ind_get(ecache)
|| edata_size_get(edata) < size
|| edata_state_get(edata)
!= ecache->state) {
assert(edata_arena_ind_get(edata) ==
ecache_ind_get(ecache));
assert(edata_state_get(edata) == extent_state_updating);
if (edata_size_get(edata) < size) {
emap_release_edata(tsdn, pac->emap, edata,
ecache->state);
edata = NULL;
}
emap_unlock_edata(tsdn, pac->emap, unlock_edata);
}
} else {
/*
@ -557,8 +548,8 @@ extent_recycle_split(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
extent_deregister_no_gdump_sub(tsdn, pac, to_leak);
extents_abandon_vm(tsdn, pac, ehooks, ecache, to_leak,
growing_retained);
assert(emap_lock_edata_from_addr(tsdn, pac->emap,
leak, false) == NULL);
assert(emap_try_acquire_edata(tsdn, pac->emap,
leak, ecache->state, true) == NULL);
}
return NULL;
}
@ -806,42 +797,11 @@ extent_alloc_wrapper(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
return edata;
}
static bool
extent_can_coalesce(ecache_t *ecache, const edata_t *inner,
const edata_t *outer) {
assert(edata_arena_ind_get(inner) == ecache_ind_get(ecache));
if (edata_arena_ind_get(inner) != edata_arena_ind_get(outer)) {
return false;
}
/*
* We wouldn't really get into this situation because one or the other
* edata would have to have a head bit set to true, but this is
* conceptually correct and cheap.
*/
if (edata_pai_get(inner) != edata_pai_get(outer)) {
return false;
}
assert(edata_state_get(inner) == extent_state_active);
if (edata_state_get(outer) != ecache->state) {
return false;
}
if (edata_committed_get(inner) != edata_committed_get(outer)) {
return false;
}
return true;
}
static bool
extent_coalesce(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, ecache_t *ecache,
edata_t *inner, edata_t *outer, bool forward, bool growing_retained) {
assert(extent_can_coalesce(ecache, inner, outer));
extent_activate_locked(tsdn, pac, ecache, outer);
extent_assert_can_coalesce(inner, outer);
eset_remove(&ecache->eset, outer);
malloc_mutex_unlock(tsdn, &ecache->mtx);
bool err = extent_merge_impl(tsdn, pac, ehooks,
@ -873,22 +833,11 @@ extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
again = false;
/* Try to coalesce forward. */
edata_t *next = emap_lock_edata_from_addr(tsdn, pac->emap,
edata_past_get(edata), inactive_only);
edata_t *next = emap_try_acquire_edata_neighbor(tsdn, pac->emap,
edata, EXTENT_PAI_PAC, ecache->state, /* forward */ true);
if (next != NULL) {
/*
* ecache->mtx only protects against races for
* like-state extents, so call extent_can_coalesce()
* before releasing next's pool lock.
*/
bool can_coalesce = extent_can_coalesce(ecache,
edata, next);
emap_unlock_edata(tsdn, pac->emap, next);
if (can_coalesce && !extent_coalesce(tsdn, pac,
ehooks, ecache, edata, next, true,
growing_retained)) {
if (!extent_coalesce(tsdn, pac, ehooks, ecache, edata,
next, true, growing_retained)) {
if (ecache->delay_coalesce) {
/* Do minimal coalescing. */
*coalesced = true;
@ -899,30 +848,11 @@ extent_try_coalesce_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
}
/* Try to coalesce backward. */
edata_t *prev = NULL;
if (edata_before_get(edata) != NULL) {
/*
* This is subtle; the rtree code asserts that its input
* pointer is non-NULL, and this is a useful thing to
* check. But it's possible that edata corresponds to
* an address of (void *)PAGE (in practice, this has
* only been observed on FreeBSD when address-space
* randomization is on, but it could in principle happen
* anywhere). In this case, edata_before_get(edata) is
* NULL, triggering the assert.
*/
prev = emap_lock_edata_from_addr(tsdn, pac->emap,
edata_before_get(edata), inactive_only);
}
edata_t *prev = emap_try_acquire_edata_neighbor(tsdn, pac->emap,
edata, EXTENT_PAI_PAC, ecache->state, /* forward */ false);
if (prev != NULL) {
bool can_coalesce = extent_can_coalesce(ecache, edata,
prev);
emap_unlock_edata(tsdn, pac->emap, prev);
if (can_coalesce && !extent_coalesce(tsdn, pac,
ehooks, ecache, edata, prev, false,
growing_retained)) {
if (!extent_coalesce(tsdn, pac, ehooks, ecache, edata,
prev, false, growing_retained)) {
edata = prev;
if (ecache->delay_coalesce) {
/* Do minimal coalescing. */
@ -1218,24 +1148,27 @@ extent_split_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks,
goto label_error_b;
}
emap_lock_edata2(tsdn, pac->emap, edata, trail);
/*
* No need to acquire trail or edata, because: 1) trail was new (just
* allocated); and 2) edata is either an active allocation (the shrink
* path), or in an acquired state (extracted from the ecache on the
* extent_recycle_split path).
*/
assert(emap_edata_is_acquired(tsdn, pac->emap, edata));
assert(emap_edata_is_acquired(tsdn, pac->emap, trail));
err = ehooks_split(tsdn, ehooks, edata_base_get(edata), size_a + size_b,
size_a, size_b, edata_committed_get(edata));
if (err) {
goto label_error_c;
goto label_error_b;
}
edata_size_set(edata, size_a);
emap_split_commit(tsdn, pac->emap, &prepare, edata, size_a, trail,
size_b);
emap_unlock_edata2(tsdn, pac->emap, edata, trail);
return trail;
label_error_c:
emap_unlock_edata2(tsdn, pac->emap, edata, trail);
label_error_b:
edata_cache_put(tsdn, pac->edata_cache, trail);
label_error_a:
@ -1277,15 +1210,15 @@ extent_merge_impl(tsdn_t *tsdn, pac_t *pac, ehooks_t *ehooks, edata_t *a,
emap_prepare_t prepare;
emap_merge_prepare(tsdn, pac->emap, &prepare, a, b);
emap_lock_edata2(tsdn, pac->emap, a, b);
assert(edata_state_get(a) == extent_state_active ||
edata_state_get(a) == extent_state_merging);
edata_state_set(a, extent_state_active);
edata_size_set(a, edata_size_get(a) + edata_size_get(b));
edata_sn_set(a, (edata_sn_get(a) < edata_sn_get(b)) ?
edata_sn_get(a) : edata_sn_get(b));
edata_zeroed_set(a, edata_zeroed_get(a) && edata_zeroed_get(b));
emap_merge_commit(tsdn, pac->emap, &prepare, a, b);
emap_unlock_edata2(tsdn, pac->emap, a, b);
edata_cache_put(tsdn, pac->edata_cache, b);

65
src/hpa_central.c
View file

@ -33,15 +33,16 @@ hpa_central_split(tsdn_t *tsdn, hpa_central_t *central, edata_t *edata,
emap_prepare_t prepare;
bool err = emap_split_prepare(tsdn, central->emap, &prepare, edata,
size, trail, cursize - size);
assert(edata_state_get(edata) == edata_state_get(trail));
if (err) {
edata_cache_small_put(tsdn, &central->ecs, trail);
return NULL;
}
emap_lock_edata2(tsdn, central->emap, edata, trail);
assert(edata_state_get(edata) == edata_state_get(trail));
edata_size_set(edata, size);
emap_split_commit(tsdn, central->emap, &prepare, edata, size, trail,
cursize - size);
emap_unlock_edata2(tsdn, central->emap, edata, trail);
return trail;
}
@ -91,7 +92,7 @@ label_success:
*/
assert(edata_state_get(edata) == extent_state_dirty);
assert(edata_base_get(edata) == edata_addr_get(edata));
emap_edata_state_update(tsdn, central->emap, edata,
emap_update_edata_state(tsdn, central->emap, edata,
extent_state_active);
return edata;
}
@ -137,43 +138,22 @@ hpa_central_alloc_grow(tsdn_t *tsdn, hpa_central_t *central,
edata_sn_set(edata, sn);
edata_sn_set(trail, sn);
emap_edata_state_update(tsdn, central->emap, trail, extent_state_dirty);
emap_update_edata_state(tsdn, central->emap, trail, extent_state_dirty);
eset_insert(&central->eset, trail);
return false;
}
static edata_t *
hpa_central_dalloc_get_merge_candidate(tsdn_t *tsdn, hpa_central_t *central,
void *addr) {
edata_t *edata = emap_lock_edata_from_addr(tsdn, central->emap, addr,
/* inactive_only */ true);
if (edata == NULL) {
return NULL;
}
extent_pai_t pai = edata_pai_get(edata);
extent_state_t state = edata_state_get(edata);
emap_unlock_edata(tsdn, central->emap, edata);
if (pai != EXTENT_PAI_HPA) {
return NULL;
}
if (state == extent_state_active) {
return NULL;
}
return edata;
}
/* Merges b into a, freeing b back to the edata cache.. */
static void
hpa_central_dalloc_merge(tsdn_t *tsdn, hpa_central_t *central, edata_t *a,
edata_t *b) {
assert(emap_edata_is_acquired(tsdn, central->emap, a));
assert(emap_edata_is_acquired(tsdn, central->emap, b));
emap_prepare_t prepare;
emap_merge_prepare(tsdn, central->emap, &prepare, a, b);
emap_lock_edata2(tsdn, central->emap, a, b);
edata_size_set(a, edata_size_get(a) + edata_size_get(b));
emap_merge_commit(tsdn, central->emap, &prepare, a, b);
emap_unlock_edata2(tsdn, central->emap, a, b);
edata_cache_small_put(tsdn, &central->ecs, b);
}
@ -189,21 +169,24 @@ hpa_central_dalloc(tsdn_t *tsdn, hpa_central_t *central, edata_t *edata) {
edata_addr_set(edata, edata_base_get(edata));
edata_zeroed_set(edata, false);
if (!edata_is_head_get(edata)) {
edata_t *lead = hpa_central_dalloc_get_merge_candidate(tsdn,
central, edata_before_get(edata));
if (lead != NULL) {
eset_remove(&central->eset, lead);
hpa_central_dalloc_merge(tsdn, central, lead, edata);
edata = lead;
}
}
edata_t *trail = hpa_central_dalloc_get_merge_candidate(tsdn, central,
edata_past_get(edata));
if (trail != NULL && !edata_is_head_get(trail)) {
/*
* Merge forward first, so that the original *edata stays active state
* for the second acquire (only necessary for sanity checking).
*/
edata_t *trail = emap_try_acquire_edata_neighbor(tsdn, central->emap,
edata, EXTENT_PAI_HPA, extent_state_dirty, /* forward */ true);
if (trail != NULL) {
eset_remove(&central->eset, trail);
hpa_central_dalloc_merge(tsdn, central, edata, trail);
}
emap_edata_state_update(tsdn, central->emap, edata, extent_state_dirty);
edata_t *lead = emap_try_acquire_edata_neighbor(tsdn, central->emap,
edata, EXTENT_PAI_HPA, extent_state_dirty, /* forward */ false);
if (lead != NULL) {
eset_remove(&central->eset, lead);
hpa_central_dalloc_merge(tsdn, central, lead, edata);
edata = lead;
}
emap_update_edata_state(tsdn, central->emap, edata, extent_state_dirty);
eset_insert(&central->eset, edata);
}