romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-04-14 14:41:42 +03:00
Code Issues Projects Releases Packages Wiki Activity

When extracting from central, hugify_eager is different than start_as_huge

Browse source
This commit is contained in:
Slobodan Predolac 2025-10-13 11:37:49 -07:00 • committed by Guangli Dai
parent 2cfa41913e
commit 3678a57c10
13 changed files with 235 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Makefile.in
View file

@ -228,6 +228,7 @@ TESTS_UNIT := \
$(srcroot)test/unit/hash.c \
$(srcroot)test/unit/hook.c \
$(srcroot)test/unit/hpa.c \
$(srcroot)test/unit/hpa_thp_always.c \
$(srcroot)test/unit/hpa_vectorized_madvise.c \
$(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \
$(srcroot)test/unit/hpa_background_thread.c \

1
include/jemalloc/internal/jemalloc_internal_externs.h
View file

@ -15,6 +15,7 @@ extern bool malloc_slow;
extern bool opt_abort;
extern bool opt_abort_conf;
extern bool opt_trust_madvise;
extern bool opt_experimental_hpa_start_huge_if_thp_always;
extern bool opt_confirm_conf;
extern bool opt_hpa;
extern hpa_shard_opts_t opt_hpa_opts;

5
src/ctl.c
View file

@ -98,6 +98,7 @@ CTL_PROTO(opt_abort_conf)
CTL_PROTO(opt_cache_oblivious)
CTL_PROTO(opt_debug_double_free_max_scan)
CTL_PROTO(opt_trust_madvise)
CTL_PROTO(opt_experimental_hpa_start_huge_if_thp_always)
CTL_PROTO(opt_confirm_conf)
CTL_PROTO(opt_hpa)
CTL_PROTO(opt_hpa_slab_max_alloc)
@ -464,6 +465,8 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
{NAME("abort_conf"), CTL(opt_abort_conf)},
{NAME("cache_oblivious"), CTL(opt_cache_oblivious)},
{NAME("trust_madvise"), CTL(opt_trust_madvise)},
{NAME("experimental_hpa_start_huge_if_thp_always"),
CTL(opt_experimental_hpa_start_huge_if_thp_always)},
{NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("hpa"), CTL(opt_hpa)},
{NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)},
{NAME("hpa_hugification_threshold"), CTL(opt_hpa_hugification_threshold)},
@ -2131,6 +2134,8 @@ CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool)
CTL_RO_NL_GEN(
opt_debug_double_free_max_scan, opt_debug_double_free_max_scan, unsigned)
CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool)
CTL_RO_NL_GEN(opt_experimental_hpa_start_huge_if_thp_always,
opt_experimental_hpa_start_huge_if_thp_always, bool)
CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
/* HPA options. */

9
src/hpa.c
View file

@ -28,6 +28,8 @@ static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
bool opt_experimental_hpa_start_huge_if_thp_always = true;
bool
hpa_hugepage_size_exceeds_limit(void) {
return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE;
@ -113,6 +115,9 @@ hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
*oom = false;
hpdata_t *ps = NULL;
bool start_as_huge = hugify_eager
|| (init_system_thp_mode == system_thp_mode_always
&& opt_experimental_hpa_start_huge_if_thp_always);
/* Is eden a perfect fit? */
if (central->eden != NULL && central->eden_len == HUGEPAGE) {
@ -122,7 +127,7 @@ hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
malloc_mutex_unlock(tsdn, &central->grow_mtx);
return NULL;
}
hpdata_init(ps, central->eden, age, hugify_eager);
hpdata_init(ps, central->eden, age, start_as_huge);
central->eden = NULL;
central->eden_len = 0;
malloc_mutex_unlock(tsdn, &central->grow_mtx);
@ -170,7 +175,7 @@ hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
assert(central->eden_len % HUGEPAGE == 0);
assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
hpdata_init(ps, central->eden, age, hugify_eager);
hpdata_init(ps, central->eden, age, start_as_huge);
char *eden_char = (char *)central->eden;
eden_char += HUGEPAGE;

6
src/jemalloc.c
View file

@ -1302,6 +1302,9 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf")
CONF_HANDLE_BOOL(opt_cache_oblivious, "cache_oblivious")
CONF_HANDLE_BOOL(opt_trust_madvise, "trust_madvise")
CONF_HANDLE_BOOL(
opt_experimental_hpa_start_huge_if_thp_always,
"experimental_hpa_start_huge_if_thp_always")
CONF_HANDLE_BOOL(
opt_huge_arena_pac_thp, "huge_arena_pac_thp")
if (strncmp("metadata_thp", k, klen) == 0) {
@ -1647,7 +1650,8 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
if (strncmp("hpa_hugify_style", k, klen) == 0) {
bool match = false;
for (int m = 0; m < hpa_hugify_style_limit; m++) {
for (int m = 0; m < hpa_hugify_style_limit;
m++) {
if (strncmp(hpa_hugify_style_names[m],
v, vlen)
== 0) {

1
src/stats.c
View file

@ -1604,6 +1604,7 @@ stats_general_print(emitter_t *emitter) {
OPT_WRITE_BOOL("abort_conf")
OPT_WRITE_BOOL("cache_oblivious")
OPT_WRITE_BOOL("confirm_conf")
OPT_WRITE_BOOL("experimental_hpa_start_huge_if_thp_always")
OPT_WRITE_BOOL("retain")
OPT_WRITE_CHAR_P("dss")
OPT_WRITE_UNSIGNED("narenas")

8
test/unit/hpa.c
View file

@ -1416,7 +1416,6 @@ TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall) {
nstime_init(&defer_curtime, 10 * 1000 * 1000);
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
/* First allocation makes the page huge */
enum { NALLOCS = HUGEPAGE_PAGES };
edata_t *edatas[NALLOCS];
ndefer_purge_calls = 0;
@ -1426,14 +1425,17 @@ TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall) {
expect_ptr_not_null(edatas[i], "Unexpected null edata");
}
hpdata_t *ps = psset_pick_alloc(&shard->psset, PAGE);
expect_false(hpdata_huge_get(ps), "Page should be non-huge");
expect_false(
hpdata_huge_get(ps), "style=none, thp=madvise, should be non-huge");
ndefer_hugify_calls = 0;
ndefer_purge_calls = 0;
hpa_shard_do_deferred_work(tsdn, shard);
expect_zu_eq(ndefer_hugify_calls, 0, "Hugify none, no syscall");
ps = psset_pick_alloc(&shard->psset, PAGE);
expect_true(ps, "Page should be huge");
expect_ptr_not_null(ps, "Unexpected null page");
expect_false(
hpdata_huge_get(ps), "style=none, thp=madvise, should be non-huge");
destroy_test_data(shard);
}

2
test/unit/hpa.sh
View file

@ -1,3 +1,3 @@
#!/bin/sh
export MALLOC_CONF="process_madvise_max_batch:0"
export MALLOC_CONF="process_madvise_max_batch:0,experimental_hpa_start_huge_if_thp_always:false"

2
test/unit/hpa_background_thread.sh
View file

@ -1,4 +1,4 @@
#!/bin/sh
export MALLOC_CONF="hpa_dirty_mult:0,hpa_min_purge_interval_ms:50,hpa_sec_nshards:0"
export MALLOC_CONF="hpa_dirty_mult:0,hpa_min_purge_interval_ms:50,hpa_sec_nshards:0,experimental_hpa_start_huge_if_thp_always:false"

202
test/unit/hpa_thp_always.c Normal file
View file

@ -0,0 +1,202 @@
#include "test/jemalloc_test.h"
#include "jemalloc/internal/hpa.h"
#include "jemalloc/internal/nstime.h"
#define SHARD_IND 111
#define ALLOC_MAX (HUGEPAGE)
typedef struct test_data_s test_data_t;
struct test_data_s {
/*
* Must be the first member -- we convert back and forth between the
* test_data_t and the hpa_shard_t;
*/
hpa_shard_t shard;
hpa_central_t central;
base_t *base;
edata_cache_t shard_edata_cache;
emap_t emap;
};
static hpa_shard_opts_t test_hpa_shard_opts_aggressive = {
/* slab_max_alloc */
HUGEPAGE,
/* hugification_threshold */
0.9 * HUGEPAGE,
/* dirty_mult */
FXP_INIT_PERCENT(11),
/* deferral_allowed */
true,
/* hugify_delay_ms */
0,
/* hugify_sync */
false,
/* min_purge_interval_ms */
5,
/* experimental_max_purge_nhp */
-1,
/* purge_threshold */
HUGEPAGE - 5 * PAGE,
/* min_purge_delay_ms */
10,
/* hugify_style */
hpa_hugify_style_eager};
static hpa_shard_t *
create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) {
bool err;
base_t *base = base_new(TSDN_NULL, /* ind */ SHARD_IND,
&ehooks_default_extent_hooks, /* metadata_use_hooks */ true);
assert_ptr_not_null(base, "");
test_data_t *test_data = malloc(sizeof(test_data_t));
assert_ptr_not_null(test_data, "");
test_data->base = base;
err = edata_cache_init(&test_data->shard_edata_cache, base);
assert_false(err, "");
err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false);
assert_false(err, "");
err = hpa_central_init(&test_data->central, test_data->base, hooks);
assert_false(err, "");
err = hpa_shard_init(&test_data->shard, &test_data->central,
&test_data->emap, test_data->base, &test_data->shard_edata_cache,
SHARD_IND, opts);
assert_false(err, "");
return (hpa_shard_t *)test_data;
}
static void
destroy_test_data(hpa_shard_t *shard) {
test_data_t *test_data = (test_data_t *)shard;
base_delete(TSDN_NULL, test_data->base);
free(test_data);
}
static uintptr_t defer_bump_ptr = HUGEPAGE * 123;
static void *
defer_test_map(size_t size) {
void *result = (void *)defer_bump_ptr;
defer_bump_ptr += size;
return result;
}
static void
defer_test_unmap(void *ptr, size_t size) {
(void)ptr;
(void)size;
}
static size_t ndefer_purge_calls = 0;
static size_t npurge_size = 0;
static void
defer_test_purge(void *ptr, size_t size) {
(void)ptr;
npurge_size = size;
++ndefer_purge_calls;
}
static bool defer_vectorized_purge_called = false;
static bool
defer_vectorized_purge(void *vec, size_t vlen, size_t nbytes) {
(void)vec;
(void)nbytes;
++ndefer_purge_calls;
defer_vectorized_purge_called = true;
return false;
}
static size_t ndefer_hugify_calls = 0;
static bool
defer_test_hugify(void *ptr, size_t size, bool sync) {
++ndefer_hugify_calls;
return false;
}
static size_t ndefer_dehugify_calls = 0;
static void
defer_test_dehugify(void *ptr, size_t size) {
++ndefer_dehugify_calls;
}
static nstime_t defer_curtime;
static void
defer_test_curtime(nstime_t *r_time, bool first_reading) {
*r_time = defer_curtime;
}
static uint64_t
defer_test_ms_since(nstime_t *past_time) {
return (nstime_ns(&defer_curtime) - nstime_ns(past_time)) / 1000 / 1000;
}
TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall_thp_always) {
test_skip_if(!hpa_supported() || (opt_process_madvise_max_batch != 0));
hpa_hooks_t hooks;
hooks.map = &defer_test_map;
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
hpa_shard_opts_t opts = test_hpa_shard_opts_aggressive;
opts.deferral_allowed = true;
opts.purge_threshold = PAGE;
opts.min_purge_delay_ms = 0;
opts.hugification_threshold = HUGEPAGE * 0.25;
opts.dirty_mult = FXP_INIT_PERCENT(10);
opts.hugify_style = hpa_hugify_style_none;
opts.min_purge_interval_ms = 0;
opts.hugify_delay_ms = 0;
hpa_shard_t *shard = create_test_data(&hooks, &opts);
bool deferred_work_generated = false;
/* Current time = 10ms */
nstime_init(&defer_curtime, 10 * 1000 * 1000);
/* Fake that system is in thp_always mode */
system_thp_mode_t old_mode = init_system_thp_mode;
init_system_thp_mode = system_thp_mode_always;
tsdn_t *tsdn = tsd_tsdn(tsd_fetch());
enum { NALLOCS = HUGEPAGE_PAGES };
edata_t *edatas[NALLOCS];
ndefer_purge_calls = 0;
for (int i = 0; i < NALLOCS / 2; i++) {
edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false,
false, false, &deferred_work_generated);
expect_ptr_not_null(edatas[i], "Unexpected null edata");
}
hpdata_t *ps = psset_pick_alloc(&shard->psset, PAGE);
expect_true(hpdata_huge_get(ps),
"Page should be huge because thp=always and hugify_style is none");
ndefer_hugify_calls = 0;
ndefer_purge_calls = 0;
hpa_shard_do_deferred_work(tsdn, shard);
expect_zu_eq(ndefer_hugify_calls, 0, "style=none, no syscall");
expect_zu_eq(ndefer_dehugify_calls, 0, "style=none, no syscall");
expect_zu_eq(ndefer_purge_calls, 1, "purge should happen");
destroy_test_data(shard);
init_system_thp_mode = old_mode;
}
TEST_END
int
main(void) {
return test_no_reentrancy(
test_hpa_hugify_style_none_huge_no_syscall_thp_always);
}

3
test/unit/hpa_thp_always.sh Normal file
View file

@ -0,0 +1,3 @@
#!/bin/sh
export MALLOC_CONF="process_madvise_max_batch:0,experimental_hpa_start_huge_if_thp_always:true"

2
test/unit/hpa_vectorized_madvise.sh
View file

@ -1,3 +1,3 @@
#!/bin/sh
export MALLOC_CONF="process_madvise_max_batch:2"
export MALLOC_CONF="process_madvise_max_batch:2,experimental_hpa_start_huge_if_thp_always:false"

2
test/unit/mallctl.c
View file

@ -300,6 +300,8 @@ TEST_BEGIN(test_mallctl_opt) {
TEST_MALLCTL_OPT(bool, abort_conf, always);
TEST_MALLCTL_OPT(bool, cache_oblivious, always);
TEST_MALLCTL_OPT(bool, trust_madvise, always);
TEST_MALLCTL_OPT(
bool, experimental_hpa_start_huge_if_thp_always, always);
TEST_MALLCTL_OPT(bool, confirm_conf, always);
TEST_MALLCTL_OPT(const char *, metadata_thp, always);
TEST_MALLCTL_OPT(bool, retain, always);