diff --git a/Makefile.in b/Makefile.in index 047e05cb..c63e6f8f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -228,6 +228,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/hash.c \ $(srcroot)test/unit/hook.c \ $(srcroot)test/unit/hpa.c \ + $(srcroot)test/unit/hpa_thp_always.c \ $(srcroot)test/unit/hpa_vectorized_madvise.c \ $(srcroot)test/unit/hpa_vectorized_madvise_large_batch.c \ $(srcroot)test/unit/hpa_background_thread.c \ diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index b502c7e7..a319dc81 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -15,6 +15,7 @@ extern bool malloc_slow; extern bool opt_abort; extern bool opt_abort_conf; extern bool opt_trust_madvise; +extern bool opt_experimental_hpa_start_huge_if_thp_always; extern bool opt_confirm_conf; extern bool opt_hpa; extern hpa_shard_opts_t opt_hpa_opts; diff --git a/src/ctl.c b/src/ctl.c index 85583bec..d3443a13 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -98,6 +98,7 @@ CTL_PROTO(opt_abort_conf) CTL_PROTO(opt_cache_oblivious) CTL_PROTO(opt_debug_double_free_max_scan) CTL_PROTO(opt_trust_madvise) +CTL_PROTO(opt_experimental_hpa_start_huge_if_thp_always) CTL_PROTO(opt_confirm_conf) CTL_PROTO(opt_hpa) CTL_PROTO(opt_hpa_slab_max_alloc) @@ -464,6 +465,8 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)}, {NAME("abort_conf"), CTL(opt_abort_conf)}, {NAME("cache_oblivious"), CTL(opt_cache_oblivious)}, {NAME("trust_madvise"), CTL(opt_trust_madvise)}, + {NAME("experimental_hpa_start_huge_if_thp_always"), + CTL(opt_experimental_hpa_start_huge_if_thp_always)}, {NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("hpa"), CTL(opt_hpa)}, {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)}, {NAME("hpa_hugification_threshold"), CTL(opt_hpa_hugification_threshold)}, @@ -2131,6 +2134,8 @@ CTL_RO_NL_GEN(opt_cache_oblivious, opt_cache_oblivious, bool) CTL_RO_NL_GEN( opt_debug_double_free_max_scan, opt_debug_double_free_max_scan, unsigned) CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool) +CTL_RO_NL_GEN(opt_experimental_hpa_start_huge_if_thp_always, + opt_experimental_hpa_start_huge_if_thp_always, bool) CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool) /* HPA options. */ diff --git a/src/hpa.c b/src/hpa.c index a7875e89..3687e6ea 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -28,6 +28,8 @@ static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self); const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"}; +bool opt_experimental_hpa_start_huge_if_thp_always = true; + bool hpa_hugepage_size_exceeds_limit(void) { return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE; @@ -113,6 +115,9 @@ hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size, *oom = false; hpdata_t *ps = NULL; + bool start_as_huge = hugify_eager + || (init_system_thp_mode == system_thp_mode_always + && opt_experimental_hpa_start_huge_if_thp_always); /* Is eden a perfect fit? */ if (central->eden != NULL && central->eden_len == HUGEPAGE) { @@ -122,7 +127,7 @@ hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size, malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); return NULL; } - hpdata_init(ps, central->eden, age, hugify_eager); + hpdata_init(ps, central->eden, age, start_as_huge); central->eden = NULL; central->eden_len = 0; malloc_mutex_unlock(tsdn, ¢ral->grow_mtx); @@ -170,7 +175,7 @@ hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size, assert(central->eden_len % HUGEPAGE == 0); assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden); - hpdata_init(ps, central->eden, age, hugify_eager); + hpdata_init(ps, central->eden, age, start_as_huge); char *eden_char = (char *)central->eden; eden_char += HUGEPAGE; diff --git a/src/jemalloc.c b/src/jemalloc.c index 72216508..0f6ff0c3 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1302,6 +1302,9 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf") CONF_HANDLE_BOOL(opt_cache_oblivious, "cache_oblivious") CONF_HANDLE_BOOL(opt_trust_madvise, "trust_madvise") + CONF_HANDLE_BOOL( + opt_experimental_hpa_start_huge_if_thp_always, + "experimental_hpa_start_huge_if_thp_always") CONF_HANDLE_BOOL( opt_huge_arena_pac_thp, "huge_arena_pac_thp") if (strncmp("metadata_thp", k, klen) == 0) { @@ -1647,7 +1650,8 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], if (strncmp("hpa_hugify_style", k, klen) == 0) { bool match = false; - for (int m = 0; m < hpa_hugify_style_limit; m++) { + for (int m = 0; m < hpa_hugify_style_limit; + m++) { if (strncmp(hpa_hugify_style_names[m], v, vlen) == 0) { diff --git a/src/stats.c b/src/stats.c index 366f96f7..4e04336e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -1604,6 +1604,7 @@ stats_general_print(emitter_t *emitter) { OPT_WRITE_BOOL("abort_conf") OPT_WRITE_BOOL("cache_oblivious") OPT_WRITE_BOOL("confirm_conf") + OPT_WRITE_BOOL("experimental_hpa_start_huge_if_thp_always") OPT_WRITE_BOOL("retain") OPT_WRITE_CHAR_P("dss") OPT_WRITE_UNSIGNED("narenas") diff --git a/test/unit/hpa.c b/test/unit/hpa.c index df2c9d96..0398e21a 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -1416,7 +1416,6 @@ TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall) { nstime_init(&defer_curtime, 10 * 1000 * 1000); tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); - /* First allocation makes the page huge */ enum { NALLOCS = HUGEPAGE_PAGES }; edata_t *edatas[NALLOCS]; ndefer_purge_calls = 0; @@ -1426,14 +1425,17 @@ TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall) { expect_ptr_not_null(edatas[i], "Unexpected null edata"); } hpdata_t *ps = psset_pick_alloc(&shard->psset, PAGE); - expect_false(hpdata_huge_get(ps), "Page should be non-huge"); + expect_false( + hpdata_huge_get(ps), "style=none, thp=madvise, should be non-huge"); ndefer_hugify_calls = 0; ndefer_purge_calls = 0; hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(ndefer_hugify_calls, 0, "Hugify none, no syscall"); ps = psset_pick_alloc(&shard->psset, PAGE); - expect_true(ps, "Page should be huge"); + expect_ptr_not_null(ps, "Unexpected null page"); + expect_false( + hpdata_huge_get(ps), "style=none, thp=madvise, should be non-huge"); destroy_test_data(shard); } diff --git a/test/unit/hpa.sh b/test/unit/hpa.sh index fe0e0b67..22451f1d 100644 --- a/test/unit/hpa.sh +++ b/test/unit/hpa.sh @@ -1,3 +1,3 @@ #!/bin/sh -export MALLOC_CONF="process_madvise_max_batch:0" +export MALLOC_CONF="process_madvise_max_batch:0,experimental_hpa_start_huge_if_thp_always:false" diff --git a/test/unit/hpa_background_thread.sh b/test/unit/hpa_background_thread.sh index 65a56a08..5c85d48b 100644 --- a/test/unit/hpa_background_thread.sh +++ b/test/unit/hpa_background_thread.sh @@ -1,4 +1,4 @@ #!/bin/sh -export MALLOC_CONF="hpa_dirty_mult:0,hpa_min_purge_interval_ms:50,hpa_sec_nshards:0" +export MALLOC_CONF="hpa_dirty_mult:0,hpa_min_purge_interval_ms:50,hpa_sec_nshards:0,experimental_hpa_start_huge_if_thp_always:false" diff --git a/test/unit/hpa_thp_always.c b/test/unit/hpa_thp_always.c new file mode 100644 index 00000000..29c86cdd --- /dev/null +++ b/test/unit/hpa_thp_always.c @@ -0,0 +1,202 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/hpa.h" +#include "jemalloc/internal/nstime.h" + +#define SHARD_IND 111 + +#define ALLOC_MAX (HUGEPAGE) + +typedef struct test_data_s test_data_t; +struct test_data_s { + /* + * Must be the first member -- we convert back and forth between the + * test_data_t and the hpa_shard_t; + */ + hpa_shard_t shard; + hpa_central_t central; + base_t *base; + edata_cache_t shard_edata_cache; + + emap_t emap; +}; + +static hpa_shard_opts_t test_hpa_shard_opts_aggressive = { + /* slab_max_alloc */ + HUGEPAGE, + /* hugification_threshold */ + 0.9 * HUGEPAGE, + /* dirty_mult */ + FXP_INIT_PERCENT(11), + /* deferral_allowed */ + true, + /* hugify_delay_ms */ + 0, + /* hugify_sync */ + false, + /* min_purge_interval_ms */ + 5, + /* experimental_max_purge_nhp */ + -1, + /* purge_threshold */ + HUGEPAGE - 5 * PAGE, + /* min_purge_delay_ms */ + 10, + /* hugify_style */ + hpa_hugify_style_eager}; + +static hpa_shard_t * +create_test_data(const hpa_hooks_t *hooks, hpa_shard_opts_t *opts) { + bool err; + base_t *base = base_new(TSDN_NULL, /* ind */ SHARD_IND, + &ehooks_default_extent_hooks, /* metadata_use_hooks */ true); + assert_ptr_not_null(base, ""); + + test_data_t *test_data = malloc(sizeof(test_data_t)); + assert_ptr_not_null(test_data, ""); + + test_data->base = base; + + err = edata_cache_init(&test_data->shard_edata_cache, base); + assert_false(err, ""); + + err = emap_init(&test_data->emap, test_data->base, /* zeroed */ false); + assert_false(err, ""); + + err = hpa_central_init(&test_data->central, test_data->base, hooks); + assert_false(err, ""); + + err = hpa_shard_init(&test_data->shard, &test_data->central, + &test_data->emap, test_data->base, &test_data->shard_edata_cache, + SHARD_IND, opts); + assert_false(err, ""); + + return (hpa_shard_t *)test_data; +} + +static void +destroy_test_data(hpa_shard_t *shard) { + test_data_t *test_data = (test_data_t *)shard; + base_delete(TSDN_NULL, test_data->base); + free(test_data); +} + +static uintptr_t defer_bump_ptr = HUGEPAGE * 123; +static void * +defer_test_map(size_t size) { + void *result = (void *)defer_bump_ptr; + defer_bump_ptr += size; + return result; +} + +static void +defer_test_unmap(void *ptr, size_t size) { + (void)ptr; + (void)size; +} + +static size_t ndefer_purge_calls = 0; +static size_t npurge_size = 0; +static void +defer_test_purge(void *ptr, size_t size) { + (void)ptr; + npurge_size = size; + ++ndefer_purge_calls; +} + +static bool defer_vectorized_purge_called = false; +static bool +defer_vectorized_purge(void *vec, size_t vlen, size_t nbytes) { + (void)vec; + (void)nbytes; + ++ndefer_purge_calls; + defer_vectorized_purge_called = true; + return false; +} + +static size_t ndefer_hugify_calls = 0; +static bool +defer_test_hugify(void *ptr, size_t size, bool sync) { + ++ndefer_hugify_calls; + return false; +} + +static size_t ndefer_dehugify_calls = 0; +static void +defer_test_dehugify(void *ptr, size_t size) { + ++ndefer_dehugify_calls; +} + +static nstime_t defer_curtime; +static void +defer_test_curtime(nstime_t *r_time, bool first_reading) { + *r_time = defer_curtime; +} + +static uint64_t +defer_test_ms_since(nstime_t *past_time) { + return (nstime_ns(&defer_curtime) - nstime_ns(past_time)) / 1000 / 1000; +} + +TEST_BEGIN(test_hpa_hugify_style_none_huge_no_syscall_thp_always) { + test_skip_if(!hpa_supported() || (opt_process_madvise_max_batch != 0)); + + hpa_hooks_t hooks; + hooks.map = &defer_test_map; + hooks.unmap = &defer_test_unmap; + hooks.purge = &defer_test_purge; + hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; + hooks.curtime = &defer_test_curtime; + hooks.ms_since = &defer_test_ms_since; + hooks.vectorized_purge = &defer_vectorized_purge; + + hpa_shard_opts_t opts = test_hpa_shard_opts_aggressive; + opts.deferral_allowed = true; + opts.purge_threshold = PAGE; + opts.min_purge_delay_ms = 0; + opts.hugification_threshold = HUGEPAGE * 0.25; + opts.dirty_mult = FXP_INIT_PERCENT(10); + opts.hugify_style = hpa_hugify_style_none; + opts.min_purge_interval_ms = 0; + opts.hugify_delay_ms = 0; + + hpa_shard_t *shard = create_test_data(&hooks, &opts); + bool deferred_work_generated = false; + /* Current time = 10ms */ + nstime_init(&defer_curtime, 10 * 1000 * 1000); + + /* Fake that system is in thp_always mode */ + system_thp_mode_t old_mode = init_system_thp_mode; + init_system_thp_mode = system_thp_mode_always; + + tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); + enum { NALLOCS = HUGEPAGE_PAGES }; + edata_t *edatas[NALLOCS]; + ndefer_purge_calls = 0; + for (int i = 0; i < NALLOCS / 2; i++) { + edatas[i] = pai_alloc(tsdn, &shard->pai, PAGE, PAGE, false, + false, false, &deferred_work_generated); + expect_ptr_not_null(edatas[i], "Unexpected null edata"); + } + hpdata_t *ps = psset_pick_alloc(&shard->psset, PAGE); + expect_true(hpdata_huge_get(ps), + "Page should be huge because thp=always and hugify_style is none"); + + ndefer_hugify_calls = 0; + ndefer_purge_calls = 0; + hpa_shard_do_deferred_work(tsdn, shard); + expect_zu_eq(ndefer_hugify_calls, 0, "style=none, no syscall"); + expect_zu_eq(ndefer_dehugify_calls, 0, "style=none, no syscall"); + expect_zu_eq(ndefer_purge_calls, 1, "purge should happen"); + + destroy_test_data(shard); + init_system_thp_mode = old_mode; +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_hpa_hugify_style_none_huge_no_syscall_thp_always); +} diff --git a/test/unit/hpa_thp_always.sh b/test/unit/hpa_thp_always.sh new file mode 100644 index 00000000..8b93006d --- /dev/null +++ b/test/unit/hpa_thp_always.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export MALLOC_CONF="process_madvise_max_batch:0,experimental_hpa_start_huge_if_thp_always:true" diff --git a/test/unit/hpa_vectorized_madvise.sh b/test/unit/hpa_vectorized_madvise.sh index c5d66afa..35d7e6b6 100644 --- a/test/unit/hpa_vectorized_madvise.sh +++ b/test/unit/hpa_vectorized_madvise.sh @@ -1,3 +1,3 @@ #!/bin/sh -export MALLOC_CONF="process_madvise_max_batch:2" +export MALLOC_CONF="process_madvise_max_batch:2,experimental_hpa_start_huge_if_thp_always:false" diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index d1974e0f..2415fda1 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -300,6 +300,8 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(bool, abort_conf, always); TEST_MALLCTL_OPT(bool, cache_oblivious, always); TEST_MALLCTL_OPT(bool, trust_madvise, always); + TEST_MALLCTL_OPT( + bool, experimental_hpa_start_huge_if_thp_always, always); TEST_MALLCTL_OPT(bool, confirm_conf, always); TEST_MALLCTL_OPT(const char *, metadata_thp, always); TEST_MALLCTL_OPT(bool, retain, always);