diff --git a/Makefile.in b/Makefile.in
index 27eb90d3..1914fc28 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -145,6 +145,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/prof_stack_range.c \
$(srcroot)src/prof_stats.c \
$(srcroot)src/prof_sys.c \
+ $(srcroot)src/prof_threshold.c \
$(srcroot)src/psset.c \
$(srcroot)src/rtree.c \
$(srcroot)src/safety_check.c \
@@ -266,6 +267,8 @@ TESTS_UNIT := \
$(srcroot)test/unit/prof_stats.c \
$(srcroot)test/unit/prof_tctx.c \
$(srcroot)test/unit/prof_thread_name.c \
+ $(srcroot)test/unit/prof_threshold.c \
+ $(srcroot)test/unit/prof_threshold_small.c \
$(srcroot)test/unit/prof_sys_thread_name.c \
$(srcroot)test/unit/psset.c \
$(srcroot)test/unit/ql.c \
diff --git a/include/jemalloc/internal/prof_externs.h b/include/jemalloc/internal/prof_externs.h
index 952ace7d..789e3811 100644
--- a/include/jemalloc/internal/prof_externs.h
+++ b/include/jemalloc/internal/prof_externs.h
@@ -11,6 +11,7 @@ extern bool opt_prof_active;
extern bool opt_prof_thread_active_init;
extern unsigned opt_prof_bt_max;
extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */
+extern size_t opt_experimental_lg_prof_threshold; /* Mean bytes between thresholds. */
extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */
extern bool opt_prof_gdump; /* High-water memory dumping. */
extern bool opt_prof_final; /* Final profile dumping. */
@@ -67,6 +68,9 @@ prof_sample_hook_t prof_sample_hook_get(void);
void prof_sample_free_hook_set(prof_sample_free_hook_t hook);
prof_sample_free_hook_t prof_sample_free_hook_get(void);
+void prof_threshold_hook_set(prof_threshold_hook_t hook);
+prof_threshold_hook_t prof_threshold_hook_get(void);
+
/* Functions only accessed in prof_inlines.h */
prof_tdata_t *prof_tdata_init(tsd_t *tsd);
prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata);
diff --git a/include/jemalloc/internal/prof_hook.h b/include/jemalloc/internal/prof_hook.h
index 087dadc6..2f3a81af 100644
--- a/include/jemalloc/internal/prof_hook.h
+++ b/include/jemalloc/internal/prof_hook.h
@@ -26,4 +26,9 @@ typedef void (*prof_sample_hook_t)(const void *ptr, size_t size, void **backtrac
/* ptr, size */
typedef void (*prof_sample_free_hook_t)(const void *, size_t);
+/*
+ * A callback hook that notifies when an allocation threshold has been crossed.
+ */
+typedef void (*prof_threshold_hook_t)(uint64_t alloc, uint64_t dealloc, uint64_t peak);
+
#endif /* JEMALLOC_INTERNAL_PROF_HOOK_H */
diff --git a/include/jemalloc/internal/prof_threshold.h b/include/jemalloc/internal/prof_threshold.h
new file mode 100644
index 00000000..dc9c8f2b
--- /dev/null
+++ b/include/jemalloc/internal/prof_threshold.h
@@ -0,0 +1,11 @@
+#ifndef JEMALLOC_INTERNAL_THRESHOLD_EVENT_H
+#define JEMALLOC_INTERNAL_THRESHOLD_EVENT_H
+
+#include "jemalloc/internal/tsd_types.h"
+
+/* The activity-triggered hooks. */
+uint64_t prof_threshold_new_event_wait(tsd_t *tsd);
+uint64_t prof_threshold_postponed_event_wait(tsd_t *tsd);
+void prof_threshold_event_handler(tsd_t *tsd, uint64_t elapsed);
+
+#endif /* JEMALLOC_INTERNAL_THRESHOLD_EVENT_H */
diff --git a/include/jemalloc/internal/thread_event.h b/include/jemalloc/internal/thread_event.h
index 46c57ed5..ad46ffe7 100644
--- a/include/jemalloc/internal/thread_event.h
+++ b/include/jemalloc/internal/thread_event.h
@@ -56,6 +56,7 @@ void tsd_te_init(tsd_t *tsd);
#define ITERATE_OVER_ALL_EVENTS \
E(tcache_gc, (opt_tcache_gc_incr_bytes > 0), true) \
E(prof_sample, (config_prof && opt_prof), true) \
+ E(prof_threshold, config_stats, true) \
E(stats_interval, (opt_stats_interval >= 0), true) \
E(tcache_gc_dalloc, (opt_tcache_gc_incr_bytes > 0), false) \
E(peak_alloc, config_stats, true) \
diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h
index 439f1d10..0ed33234 100644
--- a/include/jemalloc/internal/tsd_internals.h
+++ b/include/jemalloc/internal/tsd_internals.h
@@ -72,6 +72,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
O(tcache_gc_dalloc_event_wait, uint64_t, uint64_t) \
O(prof_sample_event_wait, uint64_t, uint64_t) \
O(prof_sample_last_event, uint64_t, uint64_t) \
+ O(prof_threshold_event_wait, uint64_t, uint64_t) \
O(stats_interval_event_wait, uint64_t, uint64_t) \
O(stats_interval_last_event, uint64_t, uint64_t) \
O(peak_alloc_event_wait, uint64_t, uint64_t) \
@@ -105,6 +106,7 @@ typedef ql_elm(tsd_t) tsd_link_t;
/* tcache_gc_dalloc_event_wait */ 0, \
/* prof_sample_event_wait */ 0, \
/* prof_sample_last_event */ 0, \
+ /* prof_threshold_event_wait */ 0, \
/* stats_interval_event_wait */ 0, \
/* stats_interval_last_event */ 0, \
/* peak_alloc_event_wait */ 0, \
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index 58bd7b3e..c43b30b1 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -83,6 +83,7 @@
+
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
index 82ad3e35..f091475e 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
@@ -133,6 +133,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 6e59c035..a195f6b3 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -83,6 +83,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
index 82ad3e35..f091475e 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
@@ -133,6 +133,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
index db06fc6d..cd16005d 100644
--- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
@@ -83,6 +83,7 @@
+
diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
index 82ad3e35..f091475e 100644
--- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
@@ -133,6 +133,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
index 01de0dcb..2d8c4be6 100644
--- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
@@ -83,6 +83,7 @@
+
diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
index 82ad3e35..f091475e 100644
--- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
@@ -133,6 +133,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/src/ctl.c b/src/ctl.c
index b0fc0487..1ebcbf8e 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -154,6 +154,7 @@ CTL_PROTO(opt_prof_active)
CTL_PROTO(opt_prof_thread_active_init)
CTL_PROTO(opt_prof_bt_max)
CTL_PROTO(opt_lg_prof_sample)
+CTL_PROTO(opt_experimental_lg_prof_threshold)
CTL_PROTO(opt_lg_prof_interval)
CTL_PROTO(opt_prof_gdump)
CTL_PROTO(opt_prof_final)
@@ -357,6 +358,7 @@ CTL_PROTO(experimental_hooks_prof_backtrace)
CTL_PROTO(experimental_hooks_prof_dump)
CTL_PROTO(experimental_hooks_prof_sample)
CTL_PROTO(experimental_hooks_prof_sample_free)
+CTL_PROTO(experimental_hooks_prof_threshold)
CTL_PROTO(experimental_hooks_safety_check_abort)
CTL_PROTO(experimental_thread_activity_callback)
CTL_PROTO(experimental_utilization_query)
@@ -539,6 +541,7 @@ static const ctl_named_node_t opt_node[] = {
{NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)},
{NAME("prof_bt_max"), CTL(opt_prof_bt_max)},
{NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)},
+ {NAME("experimental_lg_prof_threshold"), CTL(opt_experimental_lg_prof_threshold)},
{NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)},
{NAME("prof_gdump"), CTL(opt_prof_gdump)},
{NAME("prof_final"), CTL(opt_prof_final)},
@@ -965,6 +968,7 @@ static const ctl_named_node_t experimental_hooks_node[] = {
{NAME("prof_dump"), CTL(experimental_hooks_prof_dump)},
{NAME("prof_sample"), CTL(experimental_hooks_prof_sample)},
{NAME("prof_sample_free"), CTL(experimental_hooks_prof_sample_free)},
+ {NAME("prof_threshold"), CTL(experimental_hooks_prof_threshold)},
{NAME("safety_check_abort"), CTL(experimental_hooks_safety_check_abort)},
};
@@ -2317,6 +2321,7 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init,
opt_prof_thread_active_init, bool)
CTL_RO_NL_CGEN(config_prof, opt_prof_bt_max, opt_prof_bt_max, unsigned)
CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t)
+CTL_RO_NL_CGEN(config_prof, opt_experimental_lg_prof_threshold, opt_experimental_lg_prof_threshold, size_t)
CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool)
CTL_RO_NL_CGEN(config_prof, opt_prof_pid_namespace, opt_prof_pid_namespace,
bool)
@@ -3778,6 +3783,32 @@ label_return:
return ret;
}
+
+static int
+experimental_hooks_prof_threshold_ctl(tsd_t *tsd, const size_t *mib,
+ size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) {
+ int ret;
+
+ if (oldp == NULL && newp == NULL) {
+ ret = EINVAL;
+ goto label_return;
+ }
+ if (oldp != NULL) {
+ prof_threshold_hook_t old_hook =
+ prof_threshold_hook_get();
+ READ(old_hook, prof_threshold_hook_t);
+ }
+ if (newp != NULL) {
+ prof_threshold_hook_t new_hook JEMALLOC_CC_SILENCE_INIT(NULL);
+ WRITE(new_hook, prof_threshold_hook_t);
+ prof_threshold_hook_set(new_hook);
+ }
+ ret = 0;
+label_return:
+ return ret;
+}
+
+
/* For integration test purpose only. No plan to move out of experimental. */
static int
experimental_hooks_safety_check_abort_ctl(tsd_t *tsd, const size_t *mib,
diff --git a/src/jemalloc.c b/src/jemalloc.c
index 67be7681..6d2f6494 100644
--- a/src/jemalloc.c
+++ b/src/jemalloc.c
@@ -1619,6 +1619,10 @@ malloc_conf_init_helper(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS],
"lg_prof_sample", 0, (sizeof(uint64_t) << 3)
- 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
true)
+ CONF_HANDLE_SIZE_T(opt_experimental_lg_prof_threshold,
+ "experimental_lg_prof_threshold", 0, (sizeof(uint64_t) << 3)
+ - 1, CONF_DONT_CHECK_MIN, CONF_CHECK_MAX,
+ true)
CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum")
CONF_HANDLE_UNSIGNED(opt_prof_bt_max, "prof_bt_max",
1, PROF_BT_MAX_LIMIT, CONF_CHECK_MIN, CONF_CHECK_MAX,
diff --git a/src/prof_threshold.c b/src/prof_threshold.c
new file mode 100644
index 00000000..28a525fc
--- /dev/null
+++ b/src/prof_threshold.c
@@ -0,0 +1,57 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/activity_callback.h"
+#include "jemalloc/internal/prof_threshold.h"
+
+#include "jemalloc/internal/prof_externs.h"
+
+/*
+ * Update every 128MB by default.
+ */
+#define PROF_THRESHOLD_LG_WAIT_DEFAULT 27
+
+/* Logically a prof_threshold_hook_t. */
+static atomic_p_t prof_threshold_hook;
+size_t opt_experimental_lg_prof_threshold = PROF_THRESHOLD_LG_WAIT_DEFAULT;
+
+void
+prof_threshold_hook_set(prof_threshold_hook_t hook) {
+ atomic_store_p(&prof_threshold_hook, hook, ATOMIC_RELEASE);
+}
+
+prof_threshold_hook_t
+prof_threshold_hook_get(void) {
+ return (prof_threshold_hook_t)atomic_load_p(&prof_threshold_hook,
+ ATOMIC_ACQUIRE);
+}
+
+/* Invoke callback for threshold reached */
+static void
+prof_threshold_update(tsd_t *tsd) {
+ prof_threshold_hook_t prof_threshold_hook = prof_threshold_hook_get();
+ if (prof_threshold_hook == NULL) {
+ return;
+ }
+ uint64_t alloc = tsd_thread_allocated_get(tsd);
+ uint64_t dalloc = tsd_thread_deallocated_get(tsd);
+ peak_t *peak = tsd_peakp_get(tsd);
+ pre_reentrancy(tsd, NULL);
+ prof_threshold_hook(alloc, dalloc, peak->cur_max);
+ post_reentrancy(tsd);
+}
+
+uint64_t
+prof_threshold_new_event_wait(tsd_t *tsd) {
+ return 1 << opt_experimental_lg_prof_threshold;
+}
+
+uint64_t
+prof_threshold_postponed_event_wait(tsd_t *tsd) {
+ return TE_MIN_START_WAIT;
+}
+
+void
+prof_threshold_event_handler(tsd_t *tsd, uint64_t elapsed) {
+ prof_threshold_update(tsd);
+}
diff --git a/src/thread_event.c b/src/thread_event.c
index 37eb5827..a8276cd7 100644
--- a/src/thread_event.c
+++ b/src/thread_event.c
@@ -69,6 +69,11 @@ peak_dalloc_fetch_elapsed(tsd_t *tsd) {
return TE_INVALID_ELAPSED;
}
+static uint64_t
+prof_threshold_fetch_elapsed(tsd_t *tsd) {
+ return TE_INVALID_ELAPSED;
+}
+
/* Per event facilities done. */
static bool
diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c
index 6784306f..02fedaa7 100644
--- a/test/unit/mallctl.c
+++ b/test/unit/mallctl.c
@@ -319,6 +319,7 @@ TEST_BEGIN(test_mallctl_opt) {
TEST_MALLCTL_OPT(bool, prof_active, prof);
TEST_MALLCTL_OPT(unsigned, prof_bt_max, prof);
TEST_MALLCTL_OPT(ssize_t, lg_prof_sample, prof);
+ TEST_MALLCTL_OPT(ssize_t, experimental_lg_prof_threshold, prof);
TEST_MALLCTL_OPT(bool, prof_accum, prof);
TEST_MALLCTL_OPT(bool, prof_pid_namespace, prof);
TEST_MALLCTL_OPT(ssize_t, lg_prof_interval, prof);
diff --git a/test/unit/prof_threshold.c b/test/unit/prof_threshold.c
new file mode 100644
index 00000000..48e9df19
--- /dev/null
+++ b/test/unit/prof_threshold.c
@@ -0,0 +1,103 @@
+#include "test/jemalloc_test.h"
+
+/* Test config (set in reset_test_config) */
+#define ALLOC_ITERATIONS_IN_THRESHOLD 10
+uint64_t threshold_bytes = 0;
+uint64_t chunk_size = 0;
+
+/* Test globals for calblack */
+uint64_t hook_calls = 0;
+uint64_t last_peak = 0;
+uint64_t last_alloc = 0;
+uint64_t alloc_baseline = 0;
+
+void
+mock_prof_threshold_hook(uint64_t alloc, uint64_t dealloc, uint64_t peak) {
+ hook_calls++;
+ last_peak = peak;
+ last_alloc = alloc;
+}
+
+/* Need the do_write flag because NULL is a valid to_write value. */
+static void
+read_write_prof_threshold_hook(prof_threshold_hook_t *to_read, bool do_write,
+ prof_threshold_hook_t to_write) {
+ size_t hook_sz = sizeof(prof_threshold_hook_t);
+ expect_d_eq(mallctl("experimental.hooks.prof_threshold",
+ (void *)to_read, &hook_sz, do_write ? &to_write : NULL, hook_sz), 0,
+ "Unexpected prof_threshold_hook mallctl failure");
+}
+
+static void
+write_prof_threshold_hook(prof_threshold_hook_t new_hook) {
+ read_write_prof_threshold_hook(NULL, true, new_hook);
+}
+
+static prof_threshold_hook_t
+read_prof_threshold_hook() {
+ prof_threshold_hook_t hook;
+ read_write_prof_threshold_hook(&hook, false, NULL);
+ return hook;
+}
+
+static void reset_test_config() {
+ hook_calls = 0;
+ last_peak = 0;
+ alloc_baseline = last_alloc; /* We run the test multiple times */
+ last_alloc = 0;
+ threshold_bytes = 1 << opt_experimental_lg_prof_threshold;
+ chunk_size = threshold_bytes / ALLOC_ITERATIONS_IN_THRESHOLD;
+}
+
+static void expect_threshold_calls(int calls) {
+ expect_zu_eq(hook_calls, calls, "Hook called the right amount of times");
+ expect_u64_lt(last_peak, chunk_size * 2, "We allocate chunk_size at a time");
+ expect_u64_ge(last_alloc, threshold_bytes * calls + alloc_baseline, "Crosses");
+}
+
+static void allocate_chunks(int chunks) {
+ for (int i = 0; i < chunks; i++) {
+ void* p = mallocx(chunk_size, 0);
+ expect_ptr_not_null(p, "Failed to allocate");
+ free(p);
+ }
+}
+
+TEST_BEGIN(test_prof_threshold_hook) {
+ /* Test setting and reading the hook (both value and null) */
+ write_prof_threshold_hook(mock_prof_threshold_hook);
+ expect_ptr_eq(read_prof_threshold_hook(), mock_prof_threshold_hook, "Unexpected hook");
+
+ write_prof_threshold_hook(NULL);
+ expect_ptr_null(read_prof_threshold_hook(), "Hook was erased");
+
+ /* Reset everything before the test */
+ reset_test_config();
+ write_prof_threshold_hook(mock_prof_threshold_hook);
+
+ int err = mallctl("thread.peak.reset", NULL, NULL, NULL, 0);
+ expect_d_eq(err, 0, "Peak reset failed");
+
+ /* Note that since we run this test multiple times and we don't reset
+ the allocation counter, each time we offset the callback by the
+ amount we allocate over the threshold. */
+
+ /* A simple small allocation is not enough to trigger the callback */
+ allocate_chunks(1);
+ expect_zu_eq(hook_calls, 0, "Hook not called yet");
+
+ /* Enough allocations to trigger the callback */
+ allocate_chunks(ALLOC_ITERATIONS_IN_THRESHOLD);
+ expect_threshold_calls(1);
+
+ /* Enough allocations to trigger the callback again */
+ allocate_chunks(ALLOC_ITERATIONS_IN_THRESHOLD);
+ expect_threshold_calls(2);
+}
+TEST_END
+
+int
+main(void) {
+ return test(
+ test_prof_threshold_hook);
+}
diff --git a/test/unit/prof_threshold_small.c b/test/unit/prof_threshold_small.c
new file mode 100644
index 00000000..67f444b1
--- /dev/null
+++ b/test/unit/prof_threshold_small.c
@@ -0,0 +1,2 @@
+#include "test/jemalloc_test.h"
+#include "prof_threshold.c"
diff --git a/test/unit/prof_threshold_small.sh b/test/unit/prof_threshold_small.sh
new file mode 100644
index 00000000..62726069
--- /dev/null
+++ b/test/unit/prof_threshold_small.sh
@@ -0,0 +1 @@
+export MALLOC_CONF="experimental_lg_prof_threshold:22"