Add experimental_enforce_hugify

src/ctl.c

@@ -99,6 +99,7 @@ CTL_PROTO(opt_cache_oblivious)
 CTL_PROTO(opt_debug_double_free_max_scan)
 CTL_PROTO(opt_trust_madvise)
 CTL_PROTO(opt_experimental_hpa_start_huge_if_thp_always)
+CTL_PROTO(opt_experimental_hpa_enforce_hugify)
 CTL_PROTO(opt_confirm_conf)
 CTL_PROTO(opt_hpa)
 CTL_PROTO(opt_hpa_slab_max_alloc)
@@ -467,6 +468,8 @@ static const ctl_named_node_t opt_node[] = {{NAME("abort"), CTL(opt_abort)},
     {NAME("trust_madvise"), CTL(opt_trust_madvise)},
     {NAME("experimental_hpa_start_huge_if_thp_always"),
         CTL(opt_experimental_hpa_start_huge_if_thp_always)},
+    {NAME("experimental_hpa_enforce_hugify"),
+        CTL(opt_experimental_hpa_enforce_hugify)},
     {NAME("confirm_conf"), CTL(opt_confirm_conf)}, {NAME("hpa"), CTL(opt_hpa)},
     {NAME("hpa_slab_max_alloc"), CTL(opt_hpa_slab_max_alloc)},
     {NAME("hpa_hugification_threshold"), CTL(opt_hpa_hugification_threshold)},
@@ -1108,30 +1111,30 @@ ctl_arena_stats_sdmerge(
 		}
 
 		ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats
-		                          .decay_dirty.npurge,
+		                         .decay_dirty.npurge,
 		    &astats->astats.pa_shard_stats.pac_stats.decay_dirty
-		         .npurge);
+		        .npurge);
 		ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats
-		                          .decay_dirty.nmadvise,
+		                         .decay_dirty.nmadvise,
 		    &astats->astats.pa_shard_stats.pac_stats.decay_dirty
-		         .nmadvise);
+		        .nmadvise);
 		ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats
-		                          .decay_dirty.purged,
+		                         .decay_dirty.purged,
 		    &astats->astats.pa_shard_stats.pac_stats.decay_dirty
-		         .purged);
+		        .purged);
 
 		ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats
-		                          .decay_muzzy.npurge,
+		                         .decay_muzzy.npurge,
 		    &astats->astats.pa_shard_stats.pac_stats.decay_muzzy
-		         .npurge);
+		        .npurge);
 		ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats
-		                          .decay_muzzy.nmadvise,
+		                         .decay_muzzy.nmadvise,
 		    &astats->astats.pa_shard_stats.pac_stats.decay_muzzy
-		         .nmadvise);
+		        .nmadvise);
 		ctl_accum_locked_u64(&sdstats->astats.pa_shard_stats.pac_stats
-		                          .decay_muzzy.purged,
+		                         .decay_muzzy.purged,
 		    &astats->astats.pa_shard_stats.pac_stats.decay_muzzy
-		         .purged);
+		        .purged);
 
 #define OP(mtx)                                                                \
 	malloc_mutex_prof_merge(                                               \
@@ -1390,7 +1393,7 @@ ctl_refresh(tsdn_t *tsdn) {
 			    background_thread_lock);
 		} else {
 			memset(&ctl_stats->mutex_prof_data
-			            [global_prof_mutex_background_thread],
+			           [global_prof_mutex_background_thread],
 			    0, sizeof(mutex_prof_data_t));
 		}
 		/* We own ctl mutex already. */
@@ -2136,6 +2139,8 @@ CTL_RO_NL_GEN(
 CTL_RO_NL_GEN(opt_trust_madvise, opt_trust_madvise, bool)
 CTL_RO_NL_GEN(opt_experimental_hpa_start_huge_if_thp_always,
     opt_experimental_hpa_start_huge_if_thp_always, bool)
+CTL_RO_NL_GEN(opt_experimental_hpa_enforce_hugify,
+    opt_experimental_hpa_enforce_hugify, bool)
 CTL_RO_NL_GEN(opt_confirm_conf, opt_confirm_conf, bool)
 
 /* HPA options. */
@@ -3770,35 +3775,29 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail,
     arenas_i(mib[2])->astats->astats.pa_shard_stats.edata_avail, size_t)
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge,
-    locked_read_u64_unsynchronized(
-        &arenas_i(mib[2])
-             ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge),
+    locked_read_u64_unsynchronized(&arenas_i(mib[2])
+            ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.npurge),
     uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise,
-    locked_read_u64_unsynchronized(
-        &arenas_i(mib[2])
-             ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise),
+    locked_read_u64_unsynchronized(&arenas_i(mib[2])
+            ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.nmadvise),
     uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged,
-    locked_read_u64_unsynchronized(
-        &arenas_i(mib[2])
-             ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged),
+    locked_read_u64_unsynchronized(&arenas_i(mib[2])
+            ->astats->astats.pa_shard_stats.pac_stats.decay_dirty.purged),
     uint64_t)
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge,
-    locked_read_u64_unsynchronized(
-        &arenas_i(mib[2])
-             ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge),
+    locked_read_u64_unsynchronized(&arenas_i(mib[2])
+            ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.npurge),
     uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise,
-    locked_read_u64_unsynchronized(
-        &arenas_i(mib[2])
-             ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise),
+    locked_read_u64_unsynchronized(&arenas_i(mib[2])
+            ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.nmadvise),
     uint64_t)
 CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged,
-    locked_read_u64_unsynchronized(
-        &arenas_i(mib[2])
-             ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged),
+    locked_read_u64_unsynchronized(&arenas_i(mib[2])
+            ->astats->astats.pa_shard_stats.pac_stats.decay_muzzy.purged),
     uint64_t)
 
 CTL_RO_CGEN(config_stats, stats_arenas_i_base,

src/hpa.c

@@ -29,6 +29,7 @@ static uint64_t hpa_time_until_deferred_work(tsdn_t *tsdn, pai_t *self);
 const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
 
 bool opt_experimental_hpa_start_huge_if_thp_always = true;
+bool opt_experimental_hpa_enforce_hugify = false;
 
 bool
 hpa_hugepage_size_exceeds_limit(void) {
@@ -430,7 +431,7 @@ hpa_update_purge_hugify_eligibility(
 		/* Assume it is huge without the need to madvise */
 		hpa_assume_huge(tsdn, shard, ps);
 	}
-	if (hpa_is_hugify_lazy(shard)
+	if ((hpa_is_hugify_lazy(shard) || opt_experimental_hpa_enforce_hugify)
 	    && hpa_good_hugification_candidate(shard, ps)
 	    && !hpdata_huge_get(ps)) {
 		nstime_t now;
@@ -538,8 +539,9 @@ hpa_purge_actual_unlocked(
 
 static inline bool
 hpa_needs_dehugify(hpa_shard_t *shard, const hpdata_t *ps) {
-	return hpa_is_hugify_lazy(shard) && hpdata_huge_get(ps)
-	    && !hpdata_empty(ps);
+	return (hpa_is_hugify_lazy(shard)
+	           || opt_experimental_hpa_enforce_hugify)
+	    && hpdata_huge_get(ps) && !hpdata_empty(ps);
 }
 
 /* Prepare purge of one page. Return number of dirty regular pages on it
@@ -736,7 +738,7 @@ hpa_try_hugify(tsdn_t *tsdn, hpa_shard_t *shard) {
 	 * what user believes is the truth on the target system, but we won't
 	 * update nhugifies stat as system call is not being made.
 	 */
-	if (hpa_is_hugify_lazy(shard)) {
+	if (hpa_is_hugify_lazy(shard) || opt_experimental_hpa_enforce_hugify) {
 		malloc_mutex_unlock(tsdn, &shard->mtx);
 		bool err = shard->central->hooks.hugify(
 		    hpdata_addr_get(to_hugify), HUGEPAGE,

src/jemalloc.c

@@ -1305,6 +1305,8 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			CONF_HANDLE_BOOL(
 			    opt_experimental_hpa_start_huge_if_thp_always,
 			    "experimental_hpa_start_huge_if_thp_always")
+			CONF_HANDLE_BOOL(opt_experimental_hpa_enforce_hugify,
+			    "experimental_hpa_enforce_hugify")
 			CONF_HANDLE_BOOL(
 			    opt_huge_arena_pac_thp, "huge_arena_pac_thp")
 			if (strncmp("metadata_thp", k, klen) == 0) {
@@ -1554,7 +1556,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			if (strncmp("percpu_arena", k, klen) == 0) {
 				bool match = false;
 				for (int m = percpu_arena_mode_names_base;
-				     m < percpu_arena_mode_names_limit; m++) {
+				    m < percpu_arena_mode_names_limit; m++) {
 					if (strncmp(percpu_arena_mode_names[m],
 					        v, vlen)
 					    == 0) {
@@ -1651,7 +1653,7 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 			if (strncmp("hpa_hugify_style", k, klen) == 0) {
 				bool match = false;
 				for (int m = 0; m < hpa_hugify_style_limit;
-				     m++) {
+				    m++) {
 					if (strncmp(hpa_hugify_style_names[m],
 					        v, vlen)
 					    == 0) {

src/stats.c

@@ -1115,7 +1115,7 @@ stats_arena_mutexes_print(
 	CTL_LEAF_PREPARE(stats_arenas_mib, 3, "mutexes");
 
 	for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes;
-	     i++) {
+	    i++) {
 		const char *name = arena_mutex_names[i];
 		emitter_json_object_kv_begin(emitter, name);
 		mutex_stats_read_arena(
@@ -1605,6 +1605,7 @@ stats_general_print(emitter_t *emitter) {
 	OPT_WRITE_BOOL("cache_oblivious")
 	OPT_WRITE_BOOL("confirm_conf")
 	OPT_WRITE_BOOL("experimental_hpa_start_huge_if_thp_always")
+	OPT_WRITE_BOOL("experimental_hpa_enforce_hugify")
 	OPT_WRITE_BOOL("retain")
 	OPT_WRITE_CHAR_P("dss")
 	OPT_WRITE_UNSIGNED("narenas")