romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-04-14 22:51:50 +03:00
Code Issues Projects Releases Packages Wiki Activity

Refactor init_system_thp_mode and print it in malloc stats.

Browse source
This commit is contained in:
guangli-dai 2025-10-07 16:18:07 -07:00
parent ca4f2ecaa8
commit 97e8f2aafd
6 changed files with 63 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

18
include/jemalloc/internal/pages.h
View file

@ -97,8 +97,12 @@ static const bool pages_can_hugify =
#endif
;
/*
* thp_mode_t are values for opt.thp, while system_thp_mode_t is for kernel thp
* settings, i.e., init_system_thp_mode.
*/
typedef enum {
thp_mode_default = 0, /* Do not change hugepage settings. */
thp_mode_do_nothing = 0, /* Respect kernel thp settings. */
thp_mode_always = 1, /* Always set MADV_HUGEPAGE. */
thp_mode_never = 2, /* Always set MADV_NOHUGEPAGE. */
@ -106,10 +110,18 @@ typedef enum {
thp_mode_not_supported = 3 /* No THP support detected. */
} thp_mode_t;
#define THP_MODE_DEFAULT thp_mode_default
typedef enum {
system_thp_mode_madvise = 0, /* Kernel THP mode: madvise */
system_thp_mode_always = 1, /* Kernel THP mode: always */
system_thp_mode_never = 2, /* Kernel THP mode: never */
system_thp_mode_not_supported = 3 /* No THP support detected. */
} system_thp_mode_t;
#define THP_MODE_DEFAULT thp_mode_do_nothing
extern thp_mode_t opt_thp;
extern thp_mode_t init_system_thp_mode; /* Initial system wide state. */
extern system_thp_mode_t init_system_thp_mode; /* Initial system wide state. */
extern const char *const thp_mode_names[];
extern const char *const system_thp_mode_names[];
void *pages_map(void *addr, size_t size, size_t alignment, bool *commit);
void pages_unmap(void *addr, size_t size);

4
src/arena.c
View file

@ -1890,8 +1890,8 @@ arena_init_huge(tsdn_t *tsdn, arena_t *a0) {
/* Make sure that b0 thp auto-switch won't happen concurrently here. */
malloc_mutex_lock(tsdn, &b0->mtx);
(&huge_arena_pac_thp)->thp_madvise = opt_huge_arena_pac_thp
&& metadata_thp_enabled() && (opt_thp == thp_mode_default)
&& (init_system_thp_mode == thp_mode_default);
&& metadata_thp_enabled() && (opt_thp == thp_mode_do_nothing)
&& (init_system_thp_mode == system_thp_mode_madvise);
(&huge_arena_pac_thp)->auto_thp_switched =
b0->auto_thp_switched;
malloc_mutex_init(&(&huge_arena_pac_thp)->lock, "pac_thp",

4
src/base.c
View file

@ -28,8 +28,8 @@ const char *const metadata_thp_mode_names[] = {"disabled", "auto", "always"};
static inline bool
metadata_thp_madvise(void) {
return (metadata_thp_enabled()
&& (init_system_thp_mode == thp_mode_default));
return (metadata_thp_enabled() &&
(init_system_thp_mode == system_thp_mode_madvise));
}
static void *

41
src/pages.c
View file

@ -44,8 +44,10 @@ static bool os_overcommits;
const char *const thp_mode_names[] = {
"default", "always", "never", "not supported"};
thp_mode_t opt_thp = THP_MODE_DEFAULT;
thp_mode_t init_system_thp_mode;
const char *const system_thp_mode_names[] = {
"madvise", "always", "never", "not supported"};
thp_mode_t opt_thp = THP_MODE_DEFAULT;
system_thp_mode_t init_system_thp_mode;
/* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */
static bool pages_can_purge_lazy_runtime = true;
@ -778,21 +780,31 @@ os_overcommits_proc(void) {
}
#endif
static bool
pages_should_skip_set_thp_state() {
if (opt_thp == thp_mode_do_nothing
|| (opt_thp == thp_mode_always
&& init_system_thp_mode == system_thp_mode_always)
|| (opt_thp == thp_mode_never
&& init_system_thp_mode == system_thp_mode_never)) {
return true;
}
return false;
}
void
pages_set_thp_state(void *ptr, size_t size) {
if (opt_thp == thp_mode_default || opt_thp == init_system_thp_mode) {
if (pages_should_skip_set_thp_state()) {
return;
}
assert(opt_thp != thp_mode_not_supported
&& init_system_thp_mode != thp_mode_not_supported);
&& init_system_thp_mode != system_thp_mode_not_supported);
if (opt_thp == thp_mode_always
&& init_system_thp_mode != thp_mode_never) {
assert(init_system_thp_mode == thp_mode_default);
&& init_system_thp_mode == system_thp_mode_madvise) {
pages_huge_unaligned(ptr, size);
} else if (opt_thp == thp_mode_never) {
assert(init_system_thp_mode == thp_mode_default
|| init_system_thp_mode == thp_mode_always);
assert(init_system_thp_mode == system_thp_mode_madvise
|| init_system_thp_mode == system_thp_mode_always);
pages_nohuge_unaligned(ptr, size);
}
}
@ -825,16 +837,16 @@ init_thp_state(void) {
}
if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_default;
init_system_thp_mode = system_thp_mode_madvise;
} else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_always;
init_system_thp_mode = system_thp_mode_always;
} else if (strncmp(buf, sys_state_never, (size_t)nread) == 0) {
init_system_thp_mode = thp_mode_never;
init_system_thp_mode = system_thp_mode_never;
} else {
goto label_error;
}
if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) {
if (init_system_thp_mode == thp_mode_default) {
if (init_system_thp_mode == system_thp_mode_madvise) {
opt_hpa_opts.hugify_style = hpa_hugify_style_lazy;
} else {
opt_hpa_opts.hugify_style = hpa_hugify_style_none;
@ -842,14 +854,15 @@ init_thp_state(void) {
}
return;
#elif defined(JEMALLOC_HAVE_MEMCNTL)
init_system_thp_mode = thp_mode_default;
init_system_thp_mode = system_thp_mode_madvise;
if (opt_hpa_opts.hugify_style == hpa_hugify_style_auto) {
opt_hpa_opts.hugify_style = hpa_hugify_style_eager;
}
return;
#endif
label_error:
opt_thp = init_system_thp_mode = thp_mode_not_supported;
opt_thp = thp_mode_not_supported;
init_system_thp_mode = system_thp_mode_not_supported;
}
bool

16
src/stats.c
View file

@ -1516,6 +1516,22 @@ stats_general_print(emitter_t *emitter) {
#undef CONFIG_WRITE_BOOL
emitter_dict_end(emitter); /* Close "config" dict. */
/* system. */
emitter_dict_begin(emitter, "system", "System configuration");
/*
* This shows system's THP mode detected at jemalloc's init time.
* jemalloc does not re-detect the mode even if it changes after
* jemalloc's init. It is assumed that system's THP mode is stable
* during the process's lifetime and a violation could lead to
* undefined behavior.
*/
const char *thp_mode_name = system_thp_mode_names[init_system_thp_mode];
emitter_kv(emitter, "thp_mode", "system.thp_mode", emitter_type_string,
&thp_mode_name);
emitter_dict_end(emitter); /* Close "system". */
/* opt. */
#define OPT_WRITE(name, var, size, emitter_type) \
if (je_mallctl("opt." name, (void *)&var, &size, NULL, 0) == 0) { \

2
test/unit/pages.c
View file

@ -10,7 +10,7 @@ TEST_BEGIN(test_pages_huge) {
pages = pages_map(NULL, alloc_size, PAGE, &commit);
expect_ptr_not_null(pages, "Unexpected pages_map() error");
if (init_system_thp_mode == thp_mode_default) {
if (init_system_thp_mode == system_thp_mode_madvise) {
hugepage = (void *)(ALIGNMENT_CEILING(
(uintptr_t)pages, HUGEPAGE));
expect_b_ne(pages_huge(hugepage, HUGEPAGE), have_madvise_huge,