Unlike prof_sample which is supported only with profiling mode active, prof_threshold is intended to be an always-supported allocation callback with much less overhead. The usage of the threshold allows performance critical callers to change program execution based on the callback: e.g. drop caches when memory becomes high or to predict the program is about to OOM ahead of time using peak memory watermarks.

Makefile.in

@@ -145,6 +145,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
 	$(srcroot)src/prof_stack_range.c \
 	$(srcroot)src/prof_stats.c \
 	$(srcroot)src/prof_sys.c \
+	$(srcroot)src/prof_threshold.c \
 	$(srcroot)src/psset.c \
 	$(srcroot)src/rtree.c \
 	$(srcroot)src/safety_check.c \
@@ -266,6 +267,8 @@ TESTS_UNIT := \
 	$(srcroot)test/unit/prof_stats.c \
 	$(srcroot)test/unit/prof_tctx.c \
 	$(srcroot)test/unit/prof_thread_name.c \
+	$(srcroot)test/unit/prof_threshold.c \
+	$(srcroot)test/unit/prof_threshold_small.c \
 	$(srcroot)test/unit/prof_sys_thread_name.c \
 	$(srcroot)test/unit/psset.c \
 	$(srcroot)test/unit/ql.c \

include/jemalloc/internal/prof_externs.h

@@ -11,6 +11,7 @@ extern bool opt_prof_active;
 extern bool opt_prof_thread_active_init;
 extern unsigned opt_prof_bt_max;
 extern size_t opt_lg_prof_sample;    /* Mean bytes between samples. */
+extern size_t opt_experimental_lg_prof_threshold;    /* Mean bytes between thresholds. */
 extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
 extern bool opt_prof_gdump;          /* High-water memory dumping. */
 extern bool opt_prof_final;          /* Final profile dumping. */
@@ -67,6 +68,9 @@ prof_sample_hook_t prof_sample_hook_get(void);
 void prof_sample_free_hook_set(prof_sample_free_hook_t hook);
 prof_sample_free_hook_t prof_sample_free_hook_get(void);
 
+void prof_threshold_hook_set(prof_threshold_hook_t hook);
+prof_threshold_hook_t prof_threshold_hook_get(void);
+
 /* Functions only accessed in prof_inlines.h */
 prof_tdata_t *prof_tdata_init(tsd_t *tsd);
 prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);

include/jemalloc/internal/prof_hook.h

@@ -26,4 +26,9 @@ typedef void (*prof_sample_hook_t)(const void *ptr, size_t size, void **backtrac
 /* ptr, size */
 typedef void (*prof_sample_free_hook_t)(const void *, size_t);
 
+/*
+ * A callback hook that notifies when an allocation threshold has been crossed.
+ */
+typedef void (*prof_threshold_hook_t)(uint64_t alloc, uint64_t dealloc, uint64_t peak);
+
 #endif /* JEMALLOC_INTERNAL_PROF_HOOK_H */

include/jemalloc/internal/prof_threshold.h

@@ -0,0 +1,11 @@
+#ifndef JEMALLOC_INTERNAL_THRESHOLD_EVENT_H
+#define JEMALLOC_INTERNAL_THRESHOLD_EVENT_H
+
+#include "jemalloc/internal/tsd_types.h"
+
+/* The activity-triggered hooks. */
+uint64_t prof_threshold_new_event_wait(tsd_t *tsd);
+uint64_t prof_threshold_postponed_event_wait(tsd_t *tsd);
+void prof_threshold_event_handler(tsd_t *tsd, uint64_t elapsed);
+
+#endif /* JEMALLOC_INTERNAL_THRESHOLD_EVENT_H */

include/jemalloc/internal/thread_event.h

@@ -56,6 +56,7 @@ void tsd_te_init(tsd_t *tsd);
 #define ITERATE_OVER_ALL_EVENTS						\
     E(tcache_gc,		(opt_tcache_gc_incr_bytes > 0), true)	\
     E(prof_sample,		(config_prof && opt_prof), true)  	\
+    E(prof_threshold,		config_stats, true)  			\
     E(stats_interval,		(opt_stats_interval >= 0), true)   	\
     E(tcache_gc_dalloc,		(opt_tcache_gc_incr_bytes > 0), false)	\
     E(peak_alloc,		config_stats, true)			\

include/jemalloc/internal/tsd_internals.h

@@ -72,6 +72,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
     O(tcache_gc_dalloc_event_wait,	uint64_t,	uint64_t)	\
     O(prof_sample_event_wait,	uint64_t,		uint64_t)	\
     O(prof_sample_last_event,	uint64_t,		uint64_t)	\
+    O(prof_threshold_event_wait,	uint64_t,	uint64_t)	\
     O(stats_interval_event_wait,	uint64_t,	uint64_t)	\
     O(stats_interval_last_event,	uint64_t,	uint64_t)	\
     O(peak_alloc_event_wait,	uint64_t,		uint64_t)	\
@@ -105,6 +106,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
     /* tcache_gc_dalloc_event_wait */	0,				\
     /* prof_sample_event_wait */	0,				\
     /* prof_sample_last_event */	0,				\
+    /* prof_threshold_event_wait */	0,				\
     /* stats_interval_event_wait */	0,				\
     /* stats_interval_last_event */	0,				\
     /* peak_alloc_event_wait */		0,				\

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj

@@ -83,6 +83,7 @@
     <ClCompile Include="..\..\..\..\src\prof_recent.c" />
     <ClCompile Include="..\..\..\..\src\prof_stats.c" />
     <ClCompile Include="..\..\..\..\src\prof_sys.c" />
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c" />
     <ClCompile Include="..\..\..\..\src\psset.c" />
     <ClCompile Include="..\..\..\..\src\rtree.c" />
     <ClCompile Include="..\..\..\..\src\safety_check.c" />

msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters

@@ -133,6 +133,9 @@
     <ClCompile Include="..\..\..\..\src\prof_sys.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\psset.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj

@@ -83,6 +83,7 @@
     <ClCompile Include="..\..\..\..\src\prof_recent.c" />
     <ClCompile Include="..\..\..\..\src\prof_stats.c" />
     <ClCompile Include="..\..\..\..\src\prof_sys.c" />
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c" />
     <ClCompile Include="..\..\..\..\src\psset.c" />
     <ClCompile Include="..\..\..\..\src\rtree.c" />
     <ClCompile Include="..\..\..\..\src\safety_check.c" />

msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters

@@ -133,6 +133,9 @@
     <ClCompile Include="..\..\..\..\src\prof_sys.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\psset.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj

@@ -83,6 +83,7 @@
     <ClCompile Include="..\..\..\..\src\prof_recent.c" />
     <ClCompile Include="..\..\..\..\src\prof_stats.c" />
     <ClCompile Include="..\..\..\..\src\prof_sys.c" />
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c" />
     <ClCompile Include="..\..\..\..\src\psset.c" />
     <ClCompile Include="..\..\..\..\src\rtree.c" />
     <ClCompile Include="..\..\..\..\src\safety_check.c" />

msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters

@@ -133,6 +133,9 @@
     <ClCompile Include="..\..\..\..\src\prof_sys.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\psset.c">
       <Filter>Source Files</Filter>
     </ClCompile>

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj

@@ -83,6 +83,7 @@
     <ClCompile Include="..\..\..\..\src\prof_recent.c" />
     <ClCompile Include="..\..\..\..\src\prof_stats.c" />
     <ClCompile Include="..\..\..\..\src\prof_sys.c" />
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c" />
     <ClCompile Include="..\..\..\..\src\psset.c" />
     <ClCompile Include="..\..\..\..\src\rtree.c" />
     <ClCompile Include="..\..\..\..\src\safety_check.c" />

msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters

@@ -133,6 +133,9 @@
     <ClCompile Include="..\..\..\..\src\prof_sys.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\..\src\prof_threshold.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\..\..\src\psset.c">
       <Filter>Source Files</Filter>
     </ClCompile>

src/ctl.c

@@ -154,6 +154,7 @@ CTL_PROTO(opt_prof_active)
 CTL_PROTO(opt_prof_thread_active_init)
 CTL_PROTO(opt_prof_bt_max)
 CTL_PROTO(opt_lg_prof_sample)
+CTL_PROTO(opt_experimental_lg_prof_threshold)
 CTL_PROTO(opt_lg_prof_interval)
 CTL_PROTO(opt_prof_gdump)
 CTL_PROTO(opt_prof_final)
@@ -357,6 +358,7 @@ CTL_PROTO(experimental_hooks_prof_backtrace)
 CTL_PROTO(experimental_hooks_prof_dump)
 CTL_PROTO(experimental_hooks_prof_sample)
 CTL_PROTO(experimental_hooks_prof_sample_free)
+CTL_PROTO(experimental_hooks_prof_threshold)
 CTL_PROTO(experimental_hooks_safety_check_abort)
 CTL_PROTO(experimental_thread_activity_callback)
 CTL_PROTO(experimental_utilization_query)
@@ -539,6 +541,7 @@ static const ctl_named_node_t opt_node[] = {
 	{NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)},
 	{NAME("prof_bt_max"), CTL(opt_prof_bt_max)},
 	{NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)},
+	{NAME("experimental_lg_prof_threshold"), CTL(opt_experimental_lg_prof_threshold)},
 	{NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)},
 	{NAME("prof_gdump"),	CTL(opt_prof_gdump)},
 	{NAME("prof_final"),	CTL(opt_prof_final)},
@@ -965,6 +968,7 @@ static const ctl_named_node_t experimental_hooks_node[] = {
 	{NAME("prof_dump"),	CTL(experimental_hooks_prof_dump)},
 	{NAME("prof_sample"),	CTL(experimental_hooks_prof_sample)},
 	{NAME("prof_sample_free"),	CTL(experimental_hooks_prof_sample_free)},
+	{NAME("prof_threshold"),	CTL(experimental_hooks_prof_threshold)},
 	{NAME("safety_check_abort"),	CTL(experimental_hooks_safety_check_abort)},
 };
 
@@ -2317,6 +2321,7 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init,
     opt_prof_thread_active_init, bool)
 CTL_RO_NL_CGEN(config_prof, opt_prof_bt_max, opt_prof_bt_max, unsigned)
 CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t)
+CTL_RO_NL_CGEN(config_prof, opt_experimental_lg_prof_threshold, opt_experimental_lg_prof_threshold, size_t)
 CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
 CTL_RO_NL_CGEN(config_prof, opt_prof_pid_namespace, opt_prof_pid_namespace,
     bool)
@@ -3778,6 +3783,32 @@ label_return:
 	return ret;
 }
 
+
+static int
+experimental_hooks_prof_threshold_ctl(tsd_t *tsd, const size_t *mib,
+    size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+	int ret;
+
+	if (oldp == NULL && newp == NULL) {
+		ret = EINVAL;
+		goto label_return;
+	}
+	if (oldp != NULL) {
+		prof_threshold_hook_t old_hook =
+		    prof_threshold_hook_get();
+		READ(old_hook, prof_threshold_hook_t);
+	}
+	if (newp != NULL) {
+		prof_threshold_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL);
+		WRITE(new_hook, prof_threshold_hook_t);
+		prof_threshold_hook_set(new_hook);
+	}
+	ret = 0;
+label_return:
+	return ret;
+}
+
+
 /* For integration test purpose only.  No plan to move out of experimental. */
 static int
 experimental_hooks_safety_check_abort_ctl(tsd_t *tsd, const size_t *mib,

src/jemalloc.c

@@ -1619,6 +1619,10 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
 				    "lg_prof_sample", 0, (sizeof(uint64_t) << 3)
 				    - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
 				    true)
+				CONF_HANDLE_SIZE_T(opt_experimental_lg_prof_threshold,
+				    "experimental_lg_prof_threshold", 0, (sizeof(uint64_t) << 3)
+				    - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
+				    true)
 				CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
 				CONF_HANDLE_UNSIGNED(opt_prof_bt_max, "prof_bt_max",
 				    1, PROF_BT_MAX_LIMIT, CONF_CHECK_MIN, CONF_CHECK_MAX,

src/prof_threshold.c

@@ -0,0 +1,57 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/activity_callback.h"
+#include "jemalloc/internal/prof_threshold.h"
+
+#include "jemalloc/internal/prof_externs.h"
+
+/*
+ * Update every 128MB by default.
+ */
+#define PROF_THRESHOLD_LG_WAIT_DEFAULT 27
+
+/* Logically a prof_threshold_hook_t. */
+static atomic_p_t prof_threshold_hook;
+size_t opt_experimental_lg_prof_threshold = PROF_THRESHOLD_LG_WAIT_DEFAULT;
+
+void
+prof_threshold_hook_set(prof_threshold_hook_t hook) {
+	atomic_store_p(&prof_threshold_hook, hook, ATOMIC_RELEASE);
+}
+
+prof_threshold_hook_t
+prof_threshold_hook_get(void) {
+	return (prof_threshold_hook_t)atomic_load_p(&prof_threshold_hook,
+	    ATOMIC_ACQUIRE);
+}
+
+/* Invoke callback for threshold reached */
+static void
+prof_threshold_update(tsd_t *tsd) {
+	prof_threshold_hook_t prof_threshold_hook = prof_threshold_hook_get();
+	if (prof_threshold_hook == NULL) {
+		return;
+        }
+	uint64_t alloc = tsd_thread_allocated_get(tsd);
+	uint64_t dalloc = tsd_thread_deallocated_get(tsd);
+	peak_t *peak = tsd_peakp_get(tsd);
+	pre_reentrancy(tsd, NULL);
+	prof_threshold_hook(alloc, dalloc, peak->cur_max);
+	post_reentrancy(tsd);
+}
+
+uint64_t
+prof_threshold_new_event_wait(tsd_t *tsd) {
+	return 1 << opt_experimental_lg_prof_threshold;
+}
+
+uint64_t
+prof_threshold_postponed_event_wait(tsd_t *tsd) {
+	return TE_MIN_START_WAIT;
+}
+
+void
+prof_threshold_event_handler(tsd_t *tsd, uint64_t elapsed) {
+	prof_threshold_update(tsd);
+}

src/thread_event.c

@@ -69,6 +69,11 @@ peak_dalloc_fetch_elapsed(tsd_t *tsd) {
 	return TE_INVALID_ELAPSED;
 }
 
+static uint64_t
+prof_threshold_fetch_elapsed(tsd_t *tsd) {
+	return TE_INVALID_ELAPSED;
+}
+
 /* Per event facilities done. */
 
 static bool

test/unit/mallctl.c

@@ -319,6 +319,7 @@ TEST_BEGIN(test_mallctl_opt) {
 	TEST_MALLCTL_OPT(bool, prof_active, prof);
 	TEST_MALLCTL_OPT(unsigned, prof_bt_max, prof);
 	TEST_MALLCTL_OPT(ssize_t, lg_prof_sample, prof);
+	TEST_MALLCTL_OPT(ssize_t, experimental_lg_prof_threshold, prof);
 	TEST_MALLCTL_OPT(bool, prof_accum, prof);
 	TEST_MALLCTL_OPT(bool, prof_pid_namespace, prof);
 	TEST_MALLCTL_OPT(ssize_t, lg_prof_interval, prof);

test/unit/prof_threshold.c

@@ -0,0 +1,103 @@
+#include "test/jemalloc_test.h"
+
+/* Test config (set in reset_test_config) */
+#define ALLOC_ITERATIONS_IN_THRESHOLD 10
+uint64_t threshold_bytes = 0;
+uint64_t chunk_size = 0;
+
+/* Test globals for calblack */
+uint64_t hook_calls = 0;
+uint64_t last_peak = 0;
+uint64_t last_alloc = 0;
+uint64_t alloc_baseline = 0;
+
+void
+mock_prof_threshold_hook(uint64_t alloc, uint64_t dealloc, uint64_t peak) {
+	hook_calls++;
+	last_peak = peak;
+	last_alloc = alloc;
+}
+
+/* Need the do_write flag because NULL is a valid to_write value. */
+static void
+read_write_prof_threshold_hook(prof_threshold_hook_t *to_read, bool do_write,
+    prof_threshold_hook_t to_write) {
+	size_t hook_sz = sizeof(prof_threshold_hook_t);
+	expect_d_eq(mallctl("experimental.hooks.prof_threshold",
+	    (void *)to_read, &hook_sz, do_write ? &to_write : NULL, hook_sz), 0,
+	    "Unexpected prof_threshold_hook mallctl failure");
+}
+
+static void
+write_prof_threshold_hook(prof_threshold_hook_t new_hook) {
+	read_write_prof_threshold_hook(NULL, true, new_hook);
+}
+
+static prof_threshold_hook_t
+read_prof_threshold_hook() {
+	prof_threshold_hook_t hook;
+	read_write_prof_threshold_hook(&hook, false, NULL);
+	return hook;
+}
+
+static void reset_test_config() {
+	hook_calls = 0;
+	last_peak = 0;
+	alloc_baseline = last_alloc; /* We run the test multiple times */
+	last_alloc = 0;
+	threshold_bytes = 1 << opt_experimental_lg_prof_threshold;
+	chunk_size = threshold_bytes / ALLOC_ITERATIONS_IN_THRESHOLD;
+}
+
+static void expect_threshold_calls(int calls) {
+	expect_zu_eq(hook_calls, calls, "Hook called the right amount of times");
+	expect_u64_lt(last_peak, chunk_size * 2, "We allocate chunk_size at a time");
+	expect_u64_ge(last_alloc, threshold_bytes * calls + alloc_baseline, "Crosses");
+}
+
+static void allocate_chunks(int chunks) {
+	for (int i = 0; i < chunks; i++) {
+		void* p = mallocx(chunk_size, 0);
+		expect_ptr_not_null(p, "Failed to allocate");
+		free(p);
+	}
+}
+
+TEST_BEGIN(test_prof_threshold_hook) {
+	/* Test setting and reading the hook (both value and null) */
+	write_prof_threshold_hook(mock_prof_threshold_hook);
+	expect_ptr_eq(read_prof_threshold_hook(), mock_prof_threshold_hook, "Unexpected hook");
+
+	write_prof_threshold_hook(NULL);
+	expect_ptr_null(read_prof_threshold_hook(), "Hook was erased");
+
+	/* Reset everything before the test */
+	reset_test_config();
+	write_prof_threshold_hook(mock_prof_threshold_hook);
+
+	int err = mallctl("thread.peak.reset", NULL, NULL, NULL, 0);
+	expect_d_eq(err, 0, "Peak reset failed");
+
+	/* Note that since we run this test multiple times and we don't reset
+	   the allocation counter, each time we offset the callback by the
+	   amount we allocate over the threshold. */
+
+	/* A simple small allocation is not enough to trigger the callback */
+	allocate_chunks(1);
+	expect_zu_eq(hook_calls, 0, "Hook not called yet");
+
+	/* Enough allocations to trigger the callback */
+	allocate_chunks(ALLOC_ITERATIONS_IN_THRESHOLD);
+	expect_threshold_calls(1);
+
+	/* Enough allocations to trigger the callback again */
+	allocate_chunks(ALLOC_ITERATIONS_IN_THRESHOLD);
+	expect_threshold_calls(2);
+}
+TEST_END
+
+int
+main(void) {
+	return test(
+	    test_prof_threshold_hook);
+}

test/unit/prof_threshold_small.c

@@ -0,0 +1,2 @@
+#include "test/jemalloc_test.h"
+#include "prof_threshold.c"

test/unit/prof_threshold_small.sh

@@ -0,0 +1 @@
+export MALLOC_CONF="experimental_lg_prof_threshold:22"