From eac4163a950f9eb8cca0c6cd5128cf74a3bab3bb Mon Sep 17 00:00:00 2001 From: guangli-dai Date: Tue, 26 Mar 2024 14:35:29 -0700 Subject: [PATCH] Add config option limit-usize-gap and runtime option limit_usize_gap. Adding a build-time config option (--enable-limit-usize-gap) and a runtime one (limit_usize_gap) to guard the changes. When build-time config is enabled, some minor CPU overhead is expected because usize will be stored and accessed apart from index. When runtime option is also enabled (it can only be enabled with the build-time config enabled). a new usize calculation approach wil be employed. This new calculation will ceil size to the closest multiple of PAGE for all sizes larger than USIZE_GROW_SLOW_THRESHOLD instead of using the size classes. --- configure.ac | 19 +++++++++++++++++++ .../internal/jemalloc_internal_defs.h.in | 6 ++++++ .../internal/jemalloc_internal_externs.h | 1 + .../jemalloc/internal/jemalloc_preamble.h.in | 8 ++++++++ include/jemalloc/internal/sz.h | 9 +++++++++ src/ctl.c | 4 ++++ src/jemalloc.c | 12 ++++++++++++ test/unit/mallctl.c | 1 + 8 files changed, 60 insertions(+) diff --git a/configure.ac b/configure.ac index b01ff56b..a55a5a08 100644 --- a/configure.ac +++ b/configure.ac @@ -2732,6 +2732,24 @@ if test "x${have_pthread}" = "x1" -a "x${je_cv_os_unfair_lock}" != "xyes" -a \ AC_DEFINE([JEMALLOC_BACKGROUND_THREAD], [ ], [ ]) fi +dnl ============================================================================ +dnl Limit the gap between two contiguous usizes to be at most PAGE. +AC_ARG_ENABLE([limit_usize_gap], + [AS_HELP_STRING([--enable-limit-usize-gap], + [Limit the gap between two contiguous usizes])], +[if test "x$limit_usize_gap" = "xno" ; then + limit_usize_gap="0" +else + limit_usize_gap="1" +fi +], +[limit_usize_gap="0"] +) +if test "x$limit_usize_gap" = "x1" ; then + AC_DEFINE([LIMIT_USIZE_GAP], [ ]) +fi +AC_SUBST([limit_usize_gap]) + dnl ============================================================================ dnl Check for glibc malloc hooks @@ -2997,4 +3015,5 @@ AC_MSG_RESULT([cxx : ${enable_cxx}]) AC_MSG_RESULT([dss : ${enable_dss}]) AC_MSG_RESULT([tsan : ${enable_tsan}]) AC_MSG_RESULT([ubsan : ${enable_ubsan}]) +AC_MSG_RESULT([limit-usize-gap : ${limit_usize_gap}]) AC_MSG_RESULT([===============================================================================]) diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 742d599d..e76eaaf4 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -475,6 +475,12 @@ /* If defined, use __int128 for optimization. */ #undef JEMALLOC_HAVE_INT128 +/* + * If defined, the gap between any two contiguous usizes should not exceed + * PAGE. + */ +#undef LIMIT_USIZE_GAP + #include "jemalloc/internal/jemalloc_internal_overrides.h" #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index 2c6b58f7..8c6df450 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -39,6 +39,7 @@ extern atomic_zu_t zero_realloc_count; extern bool opt_cache_oblivious; extern unsigned opt_debug_double_free_max_scan; extern size_t opt_calloc_madvise_threshold; +extern bool opt_limit_usize_gap; extern const char *opt_malloc_conf_symlink; extern const char *opt_malloc_conf_env_var; diff --git a/include/jemalloc/internal/jemalloc_preamble.h.in b/include/jemalloc/internal/jemalloc_preamble.h.in index a59c3489..ef637a2d 100644 --- a/include/jemalloc/internal/jemalloc_preamble.h.in +++ b/include/jemalloc/internal/jemalloc_preamble.h.in @@ -276,4 +276,12 @@ static const bool have_memcntl = #endif ; +static const bool config_limit_usize_gap = +#ifdef LIMIT_USIZE_GAP + true +#else + false +#endif + ; + #endif /* JEMALLOC_PREAMBLE_H */ diff --git a/include/jemalloc/internal/sz.h b/include/jemalloc/internal/sz.h index a2d2debc..5c8e3cb1 100644 --- a/include/jemalloc/internal/sz.h +++ b/include/jemalloc/internal/sz.h @@ -54,6 +54,15 @@ extern size_t sz_large_pad; extern void sz_boot(const sc_data_t *sc_data, bool cache_oblivious); +JEMALLOC_ALWAYS_INLINE bool +sz_limit_usize_gap_enabled() { +#ifdef LIMIT_USIZE_GAP + return opt_limit_usize_gap; +#else + return false; +#endif +} + JEMALLOC_ALWAYS_INLINE pszind_t sz_psz2ind(size_t psz) { assert(psz > 0); diff --git a/src/ctl.c b/src/ctl.c index 1ebcbf8e..4be491e9 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -168,6 +168,7 @@ CTL_PROTO(opt_prof_sys_thread_name) CTL_PROTO(opt_prof_time_res) CTL_PROTO(opt_lg_san_uaf_align) CTL_PROTO(opt_zero_realloc) +CTL_PROTO(opt_limit_usize_gap) CTL_PROTO(opt_malloc_conf_symlink) CTL_PROTO(opt_malloc_conf_env_var) CTL_PROTO(opt_malloc_conf_global_var) @@ -557,6 +558,7 @@ static const ctl_named_node_t opt_node[] = { {NAME("zero_realloc"), CTL(opt_zero_realloc)}, {NAME("debug_double_free_max_scan"), CTL(opt_debug_double_free_max_scan)}, + {NAME("limit_usize_gap"), CTL(opt_limit_usize_gap)}, {NAME("malloc_conf"), CHILD(named, opt_malloc_conf)} }; @@ -2341,6 +2343,8 @@ CTL_RO_NL_CGEN(config_uaf_detection, opt_lg_san_uaf_align, opt_lg_san_uaf_align, ssize_t) CTL_RO_NL_GEN(opt_zero_realloc, zero_realloc_mode_names[opt_zero_realloc_action], const char *) +CTL_RO_NL_CGEN(config_limit_usize_gap, opt_limit_usize_gap, opt_limit_usize_gap, + bool) /* malloc_conf options */ CTL_RO_NL_CGEN(opt_malloc_conf_symlink, opt_malloc_conf_symlink, diff --git a/src/jemalloc.c b/src/jemalloc.c index 8ae72efb..e313de36 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -123,6 +123,13 @@ zero_realloc_action_t opt_zero_realloc_action = atomic_zu_t zero_realloc_count = ATOMIC_INIT(0); +bool opt_limit_usize_gap = +#ifdef LIMIT_USIZE_GAP + false; +#else + false; +#endif + const char *const zero_realloc_mode_names[] = { "alloc", "free", @@ -1763,6 +1770,11 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], "san_guard_large", 0, SIZE_T_MAX, CONF_DONT_CHECK_MIN, CONF_DONT_CHECK_MAX, false) + if (config_limit_usize_gap) { + CONF_HANDLE_BOOL(opt_limit_usize_gap, + "limit_usize_gap"); + } + CONF_ERROR("Invalid conf pair", k, klen, v, vlen); #undef CONF_ERROR #undef CONF_CONTINUE diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 02fedaa7..296b7bff 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -332,6 +332,7 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(bool, prof_sys_thread_name, prof); TEST_MALLCTL_OPT(ssize_t, lg_san_uaf_align, uaf_detection); TEST_MALLCTL_OPT(unsigned, debug_double_free_max_scan, always); + TEST_MALLCTL_OPT(bool, limit_usize_gap, limit_usize_gap); #undef TEST_MALLCTL_OPT }