[thread_event] Add support for user events in thread events when stats are enabled

include/jemalloc/internal/tcache_externs.h

@@ -82,14 +82,6 @@ void tcache_enabled_set(tsd_t *tsd, bool enabled);
 
 void tcache_assert_initialized(tcache_t *tcache);
 
-/* Only accessed by thread event. */
-uint64_t tcache_gc_new_event_wait(tsd_t *tsd);
-uint64_t tcache_gc_postponed_event_wait(tsd_t *tsd);
-void tcache_gc_event_handler(tsd_t *tsd, uint64_t elapsed);
-uint64_t tcache_gc_dalloc_new_event_wait(tsd_t *tsd);
-uint64_t tcache_gc_dalloc_postponed_event_wait(tsd_t *tsd);
-void tcache_gc_dalloc_event_handler(tsd_t *tsd, uint64_t elapsed);
-
 extern te_base_cb_t tcache_gc_te_handler;
 
 #endif /* JEMALLOC_INTERNAL_TCACHE_EXTERNS_H */

include/jemalloc/internal/thread_event.h

@@ -48,10 +48,12 @@ void te_assert_invariants_debug(tsd_t *tsd);
 void te_event_trigger(tsd_t *tsd, te_ctx_t *ctx);
 void te_recompute_fast_threshold(tsd_t *tsd);
 void tsd_te_init(tsd_t *tsd);
+void te_adjust_thresholds_helper(tsd_t *tsd, te_ctx_t *ctx,
+    uint64_t wait);
 
 /* List of all thread event counters. */
-#define ITERATE_OVER_ALL_COUNTERS                                       \
-        C(thread_allocated)						\
+#define ITERATE_OVER_ALL_COUNTERS					\
+	C(thread_allocated)						\
 	C(thread_allocated_last_event)					\
 	C(prof_sample_last_event)					\
 	C(stats_interval_last_event)

include/jemalloc/internal/thread_event_registry.h

@@ -2,37 +2,41 @@
 #define JEMALLOC_INTERNAL_THREAD_EVENT_REGISTRY_H
 
 #include "jemalloc/internal/jemalloc_preamble.h"
-#include "jemalloc/internal/tsd.h"
+#include "jemalloc/internal/tsd_types.h"
+
+#define TE_MAX_USER_EVENTS 4
 
 /* "te" is short for "thread_event" */
 enum te_alloc_e {
 #ifdef JEMALLOC_PROF
-    te_alloc_prof_sample,
+	te_alloc_prof_sample,
 #endif
-    te_alloc_stats_interval,
+	te_alloc_stats_interval,
+	te_alloc_tcache_gc,
 #ifdef JEMALLOC_STATS
-    te_alloc_prof_threshold,
+	te_alloc_prof_threshold,
+	te_alloc_peak,
 #endif
-    te_alloc_tcache_gc,
-#ifdef JEMALLOC_STATS
-    te_alloc_peak,
-    te_alloc_last = te_alloc_peak,
-#else
-    te_alloc_last = te_alloc_tcache_gc,
-#endif
-    te_alloc_count = te_alloc_last + 1
+	te_alloc_user0,
+	te_alloc_user1,
+	te_alloc_user2,
+	te_alloc_user3,
+	te_alloc_last = te_alloc_user3,
+	te_alloc_count = te_alloc_last + 1
 };
 typedef enum te_alloc_e te_alloc_t;
 
 enum te_dalloc_e {
-    te_dalloc_tcache_gc,
+	te_dalloc_tcache_gc,
 #ifdef JEMALLOC_STATS
-    te_dalloc_peak,
-    te_dalloc_last = te_dalloc_peak,
-#else
-    te_dalloc_last = te_dalloc_tcache_gc,
+	te_dalloc_peak,
 #endif
-    te_dalloc_count = te_dalloc_last + 1
+	te_dalloc_user0,
+	te_dalloc_user1,
+	te_dalloc_user2,
+	te_dalloc_user3,
+	te_dalloc_last = te_dalloc_user3,
+	te_dalloc_count = te_dalloc_last + 1
 };
 typedef enum te_dalloc_e te_dalloc_t;
 
@@ -42,17 +46,63 @@ struct te_data_s {
 	uint64_t alloc_wait[te_alloc_count];
 	uint64_t dalloc_wait[te_dalloc_count];
 };
-#define TE_DATA_INITIALIZER { {0}, {0} }
+#define TE_DATA_INITIALIZER                                                    \
+	{                                                                      \
+		{0}, {                                                         \
+			0                                                      \
+		}                                                              \
+	}
+
+/*
+ * Check if user event is installed, installed and enabled, or not
+ * installed.
+ *
+ */
+enum te_enabled_e { te_enabled_not_installed, te_enabled_yes, te_enabled_no };
+typedef enum te_enabled_e te_enabled_t;
 
 typedef struct te_base_cb_s te_base_cb_t;
 struct te_base_cb_s {
-    bool (*enabled)(void);
-    uint64_t (*new_event_wait)(tsd_t *tsd);
-    uint64_t (*postponed_event_wait)(tsd_t *tsd);
-    void (*event_handler)(tsd_t *tsd);
+	te_enabled_t (*enabled)(void);
+	uint64_t (*new_event_wait)(tsd_t *tsd);
+	uint64_t (*postponed_event_wait)(tsd_t *tsd);
+	void (*event_handler)(tsd_t *tsd);
 };
 
 extern te_base_cb_t *te_alloc_handlers[te_alloc_count];
 extern te_base_cb_t *te_dalloc_handlers[te_dalloc_count];
 
+bool experimental_thread_events_boot(void);
+
+/*
+ *  User callback for thread events
+ *
+ *  is_alloc - true if event is allocation, false if event is free
+ *  tallocated  - number of bytes allocated on current thread so far
+ *  tdallocated - number of bytes allocated on current thread so far
+ */
+typedef void (*user_event_cb_t)(
+    bool is_alloc, uint64_t tallocated, uint64_t tdallocated);
+
+typedef struct user_hook_object_s user_hook_object_t;
+struct user_hook_object_s {
+	user_event_cb_t callback;
+	uint64_t interval;
+	bool is_alloc_only;
+};
+
+/*
+ * register user callback
+ *
+ * return zero if event was registered
+ *
+ * if interval is zero or callback is NULL, or
+ * no more slots are available event will not be registered
+ * and non-zero value will be returned
+ *
+ */
+int te_register_user_handler(tsdn_t *tsdn, user_hook_object_t *te_uobj);
+
+te_enabled_t te_user_event_enabled(size_t ue_idx, bool is_alloc);
+
 #endif /* JEMALLOC_INTERNAL_THREAD_EVENT_REGISTRY_H */

include/jemalloc/internal/witness.h

@@ -78,6 +78,7 @@ enum witness_rank_e {
 	WITNESS_RANK_PROF_RECENT_ALLOC = WITNESS_RANK_LEAF,
 	WITNESS_RANK_PROF_STATS = WITNESS_RANK_LEAF,
 	WITNESS_RANK_PROF_THREAD_ACTIVE_INIT = WITNESS_RANK_LEAF,
+	WITNESS_RANK_THREAD_EVENTS_USER = WITNESS_RANK_LEAF,
 };
 typedef enum witness_rank_e witness_rank_t;