romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-08-26 00:53:34 +03:00
Code Issues Projects Releases Packages Wiki Activity

Allow chunks to not be naturally aligned.

Browse source 
Precisely size extents for huge size classes that aren't multiples of
chunksize.
This commit is contained in:
Jason Evans 2016-05-26 22:12:38 -07:00
parent 741967e79d
commit 4731cd47f7
11 changed files with 105 additions and 268 deletions
Download patch file Download diff file Expand all files Collapse all files

145
src/huge.c
View file

@ -9,7 +9,7 @@ huge_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero)
assert(usize == s2u(usize));
return (huge_palloc(tsdn, arena, usize, chunksize, zero));
return (huge_palloc(tsdn, arena, usize, PAGE, zero));
}
void *
@ -20,14 +20,11 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
extent_t *extent;
bool is_zeroed;
/* Allocate one or more contiguous chunks for this request. */
assert(!tsdn_null(tsdn) || arena != NULL);
ausize = sa2u(usize, alignment);
if (unlikely(ausize == 0 || ausize > HUGE_MAXCLASS))
return (NULL);
assert(ausize >= chunksize);
/*
* Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that
@ -40,9 +37,6 @@ huge_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
arena, usize, alignment, &is_zeroed)) == NULL)
return (NULL);
if (usize < extent_size_get(extent))
extent_size_set(extent, usize);
/* Insert extent into huge. */
malloc_mutex_lock(tsdn, &arena->huge_mtx);
ql_elm_new(extent, ql_link);
@ -86,80 +80,20 @@ huge_dalloc_junk(tsdn_t *tsdn, void *ptr, size_t usize)
huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl);
#endif
static void
huge_ralloc_no_move_similar(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
size_t usize_max, bool zero)
{
size_t usize, usize_next;
arena_t *arena = extent_arena_get(extent);
size_t oldsize = extent_size_get(extent);
chunk_hooks_t chunk_hooks = CHUNK_HOOKS_INITIALIZER;
bool pre_zeroed, post_zeroed;
/* Increase usize to incorporate extra. */
for (usize = usize_min; usize < usize_max && (usize_next = s2u(usize+1))
<= oldsize; usize = usize_next)
; /* Do nothing. */
if (oldsize == usize)
return;
pre_zeroed = extent_zeroed_get(extent);
/* Fill if necessary (shrinking). */
if (oldsize > usize) {
size_t sdiff = oldsize - usize;
if (config_fill && unlikely(opt_junk_free)) {
memset((void *)((uintptr_t)extent_addr_get(extent) +
usize), JEMALLOC_FREE_JUNK, sdiff);
post_zeroed = false;
} else {
post_zeroed = !chunk_purge_wrapper(tsdn, arena,
&chunk_hooks, extent, usize, sdiff);
}
} else
post_zeroed = pre_zeroed;
/* Update the size of the huge allocation. */
assert(extent_size_get(extent) != usize);
malloc_mutex_lock(tsdn, &arena->huge_mtx);
extent_size_set(extent, usize);
malloc_mutex_unlock(tsdn, &arena->huge_mtx);
/* Update zeroed. */
extent_zeroed_set(extent, post_zeroed);
arena_chunk_ralloc_huge_similar(tsdn, arena, extent, oldsize);
/* Fill if necessary (growing). */
if (oldsize < usize) {
if (zero || (config_fill && unlikely(opt_zero))) {
if (!pre_zeroed) {
memset((void *)
((uintptr_t)extent_addr_get(extent) +
oldsize), 0, usize - oldsize);
}
} else if (config_fill && unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)extent_addr_get(extent) +
oldsize), JEMALLOC_ALLOC_JUNK, usize - oldsize);
}
}
}
static bool
huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
{
arena_t *arena = extent_arena_get(extent);
size_t oldsize = extent_size_get(extent);
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena);
size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize);
size_t sdiff = CHUNK_CEILING(usize) - usize;
size_t diff = oldsize - usize;
assert(oldsize > usize);
/* Split excess chunks. */
if (cdiff != 0) {
/* Split excess pages. */
if (diff != 0) {
extent_t *trail = chunk_split_wrapper(tsdn, arena, &chunk_hooks,
extent, CHUNK_CEILING(usize), cdiff);
extent, usize, diff);
if (trail == NULL)
return (true);
@ -171,28 +105,6 @@ huge_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize)
arena_chunk_cache_dalloc(tsdn, arena, &chunk_hooks, trail);
}
/* Optionally fill trailing subchunk. */
if (sdiff != 0) {
bool post_zeroed;
if (config_fill && unlikely(opt_junk_free)) {
huge_dalloc_junk(tsdn,
(void *)((uintptr_t)extent_addr_get(extent) +
usize), sdiff);
post_zeroed = false;
} else {
post_zeroed = !chunk_purge_wrapper(tsdn, arena,
&chunk_hooks, extent, usize, sdiff);
if (config_fill && unlikely(opt_zero) && !post_zeroed) {
memset((void *)
((uintptr_t)extent_addr_get(extent) +
usize), 0, sdiff);
}
}
extent_zeroed_set(extent, post_zeroed);
}
arena_chunk_ralloc_huge_shrink(tsdn, arena, extent, oldsize);
return (false);
@ -204,20 +116,18 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
{
arena_t *arena = extent_arena_get(extent);
size_t oldsize = extent_size_get(extent);
bool is_zeroed_subchunk = extent_zeroed_get(extent);
bool is_zeroed_chunk = false;
bool is_zeroed_trail = false;
chunk_hooks_t chunk_hooks = chunk_hooks_get(tsdn, arena);
size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize);
void *nchunk =
(void *)CHUNK_CEILING((uintptr_t)extent_past_get(extent));
size_t trailsize = usize - oldsize;
extent_t *trail;
if ((trail = arena_chunk_cache_alloc(tsdn, arena, &chunk_hooks, nchunk,
cdiff, chunksize, &is_zeroed_chunk)) == NULL) {
if ((trail = arena_chunk_cache_alloc(tsdn, arena, &chunk_hooks,
extent_past_get(extent), trailsize, PAGE, &is_zeroed_trail)) ==
NULL) {
bool commit = true;
if ((trail = chunk_alloc_wrapper(tsdn, arena, &chunk_hooks,
nchunk, cdiff, chunksize, &is_zeroed_chunk, &commit, false))
== NULL)
extent_past_get(extent), trailsize, PAGE, &is_zeroed_trail,
&commit, false)) == NULL)
return (true);
}
@ -227,23 +137,15 @@ huge_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize,
}
if (zero || (config_fill && unlikely(opt_zero))) {
if (!is_zeroed_subchunk) {
if (!is_zeroed_trail) {
memset((void *)((uintptr_t)extent_addr_get(extent) +
oldsize), 0, CHUNK_CEILING(oldsize) - oldsize);
}
if (!is_zeroed_chunk) {
memset((void *)((uintptr_t)extent_addr_get(extent) +
CHUNK_CEILING(oldsize)), 0, usize -
CHUNK_CEILING(oldsize));
oldsize), 0, usize - oldsize);
}
} else if (config_fill && unlikely(opt_junk_alloc)) {
memset((void *)((uintptr_t)extent_addr_get(extent) + oldsize),
JEMALLOC_ALLOC_JUNK, usize - oldsize);
}
if (usize < extent_size_get(extent))
extent_size_set(extent, usize);
arena_chunk_ralloc_huge_expand(tsdn, arena, extent, oldsize);
return (false);
@ -260,7 +162,7 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
/* Both allocation sizes must be huge to avoid a move. */
assert(extent_size_get(extent) >= chunksize && usize_max >= chunksize);
if (CHUNK_CEILING(usize_max) > CHUNK_CEILING(extent_size_get(extent))) {
if (usize_max > extent_size_get(extent)) {
/* Attempt to expand the allocation in-place. */
if (!huge_ralloc_no_move_expand(tsdn, extent, usize_max,
zero)) {
@ -268,9 +170,9 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
return (false);
}
/* Try again, this time with usize_min. */
if (usize_min < usize_max && CHUNK_CEILING(usize_min) >
CHUNK_CEILING(extent_size_get(extent)) &&
huge_ralloc_no_move_expand(tsdn, extent, usize_min, zero)) {
if (usize_min < usize_max && usize_min > extent_size_get(extent)
&& huge_ralloc_no_move_expand(tsdn, extent, usize_min,
zero)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
@ -280,17 +182,14 @@ huge_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min,
* Avoid moving the allocation if the existing chunk size accommodates
* the new size.
*/
if (CHUNK_CEILING(extent_size_get(extent)) >= CHUNK_CEILING(usize_min)
&& CHUNK_CEILING(extent_size_get(extent)) <=
CHUNK_CEILING(usize_max)) {
huge_ralloc_no_move_similar(tsdn, extent, usize_min, usize_max,
zero);
if (extent_size_get(extent) >= usize_min && extent_size_get(extent) <=
usize_max) {
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
}
/* Attempt to shrink the allocation in-place. */
if (CHUNK_CEILING(extent_size_get(extent)) > CHUNK_CEILING(usize_max)) {
if (extent_size_get(extent) > usize_max) {
if (!huge_ralloc_no_move_shrink(tsdn, extent, usize_max)) {
arena_decay_tick(tsdn, extent_arena_get(extent));
return (false);
@ -304,7 +203,7 @@ huge_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
size_t alignment, bool zero)
{
if (alignment <= chunksize)
if (alignment <= PAGE)
return (huge_malloc(tsdn, arena, usize, zero));
return (huge_palloc(tsdn, arena, usize, alignment, zero));
}