diff --git a/Makefile.in b/Makefile.in
index f939350f..13bc5a24 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -132,6 +132,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/inspect.c \
$(srcroot)src/large.c \
$(srcroot)src/log.c \
+ $(srcroot)src/malloc_dispatch.c \
$(srcroot)src/malloc_io.c \
$(srcroot)src/conf.c \
$(srcroot)src/mutex.c \
diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h
index 06189d56..694c5d81 100644
--- a/include/jemalloc/internal/arena_externs.h
+++ b/include/jemalloc/internal/arena_externs.h
@@ -70,12 +70,9 @@ cache_bin_sz_t arena_ptr_array_fill_small(tsdn_t *tsdn, arena_t *arena,
void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
bool zero, bool slab);
-void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
- bool zero, bool slab, tcache_t *tcache);
void arena_prof_promote(
tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize);
-void arena_dalloc_promoted(
- tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
+size_t arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr);
void arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, edata_t *slab);
void arena_dalloc_small(tsdn_t *tsdn, void *ptr);
@@ -84,8 +81,6 @@ void arena_ptr_array_flush(tsd_t *tsd, szind_t binind,
arena_t *stats_arena, cache_bin_stats_t merge_stats);
bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
size_t extra, bool zero, size_t *newsize);
-void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
- size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache);
dss_prec_t arena_dss_prec_get(const arena_t *arena);
ehooks_t *arena_get_ehooks(const arena_t *arena);
extent_hooks_t *arena_set_extent_hooks(
diff --git a/include/jemalloc/internal/arena_inlines_b.h b/include/jemalloc/internal/arena_inlines_b.h
index 64957c7b..005f422e 100644
--- a/include/jemalloc/internal/arena_inlines_b.h
+++ b/include/jemalloc/internal/arena_inlines_b.h
@@ -17,7 +17,6 @@
#include "jemalloc/internal/safety_check.h"
#include "jemalloc/internal/sc.h"
#include "jemalloc/internal/sz.h"
-#include "jemalloc/internal/tcache_inlines.h"
#include "jemalloc/internal/ticker.h"
static inline arena_t *
@@ -193,26 +192,6 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) {
arena_decay_ticks(tsdn, arena, 1);
}
-JEMALLOC_ALWAYS_INLINE void *
-arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero,
- bool slab, tcache_t *tcache, bool slow_path) {
- assert(!tsdn_null(tsdn) || tcache == NULL);
-
- if (likely(tcache != NULL)) {
- if (likely(slab)) {
- assert(sz_can_use_slab(size));
- return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
- size, ind, zero, slow_path);
- } else if (likely(tcache_can_cache_large(tcache, ind))) {
- return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
- size, ind, zero, slow_path);
- }
- /* (size > tcache_max) case falls through. */
- }
-
- return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
-}
-
JEMALLOC_ALWAYS_INLINE arena_t *
arena_aalloc(tsdn_t *tsdn, const void *ptr) {
edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
@@ -261,244 +240,6 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) {
return edata_usize_get(full_alloc_ctx.edata);
}
-static inline void
-arena_dalloc_large_no_tcache(
- tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
- /*
- * szind is still needed in this function mainly becuase
- * szind < SC_NBINS determines not only if this is a small alloc,
- * but also if szind is valid (an inactive extent would have
- * szind == SC_NSIZES).
- */
- if (config_prof && unlikely(szind < SC_NBINS)) {
- arena_dalloc_promoted(tsdn, ptr, NULL, true);
- } else {
- edata_t *edata = emap_edata_lookup(
- tsdn, &arena_emap_global, ptr);
- if (large_dalloc_safety_checks(edata, ptr, usize)) {
- /* See the comment in isfree. */
- return;
- }
- large_dalloc(tsdn, edata);
- }
-}
-
-static inline void
-arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
- assert(ptr != NULL);
-
- emap_alloc_ctx_t alloc_ctx;
- emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
- if (config_debug) {
- edata_t *edata = emap_edata_lookup(
- tsdn, &arena_emap_global, ptr);
- assert(alloc_ctx.szind == edata_szind_get(edata));
- assert(alloc_ctx.szind < SC_NSIZES);
- assert(alloc_ctx.slab == edata_slab_get(edata));
- assert(emap_alloc_ctx_usize_get(&alloc_ctx)
- == edata_usize_get(edata));
- }
-
- if (likely(alloc_ctx.slab)) {
- /* Small allocation. */
- arena_dalloc_small(tsdn, ptr);
- } else {
- arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
- emap_alloc_ctx_usize_get(&alloc_ctx));
- }
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache, szind_t szind,
- size_t usize, bool slow_path) {
- assert(!tsdn_null(tsdn) && tcache != NULL);
- bool is_sample_promoted = config_prof && szind < SC_NBINS;
- if (unlikely(is_sample_promoted)) {
- arena_dalloc_promoted(tsdn, ptr, tcache, slow_path);
- } else {
- if (tcache_can_cache_large(tcache, szind)) {
- tcache_dalloc_large(
- tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
- } else {
- edata_t *edata = emap_edata_lookup(
- tsdn, &arena_emap_global, ptr);
- if (large_dalloc_safety_checks(edata, ptr, usize)) {
- /* See the comment in isfree. */
- return;
- }
- large_dalloc(tsdn, edata);
- }
- }
-}
-
-JEMALLOC_ALWAYS_INLINE bool
-arena_tcache_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
- if (!config_debug) {
- return false;
- }
- edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
- szind_t binind = edata_szind_get(edata);
- div_info_t div_info = arena_binind_div_info[binind];
- /*
- * Calls the internal function bin_slab_regind_impl because the
- * safety check does not require a lock.
- */
- size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
- slab_data_t *slab_data = edata_slab_data_get(edata);
- const bin_info_t *bin_info = &bin_infos[binind];
- assert(edata_nfree_get(edata) < bin_info->nregs);
- if (unlikely(!bitmap_get(
- slab_data->bitmap, &bin_info->bitmap_info, regind))) {
- safety_check_fail(
- "Invalid deallocation detected: the pointer being freed (%p) not "
- "currently active, possibly caused by double free bugs.\n",
- ptr);
- return true;
- }
- return false;
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
- emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
- assert(!tsdn_null(tsdn) || tcache == NULL);
- assert(ptr != NULL);
-
- if (unlikely(tcache == NULL)) {
- arena_dalloc_no_tcache(tsdn, ptr);
- return;
- }
-
- emap_alloc_ctx_t alloc_ctx;
- if (caller_alloc_ctx != NULL) {
- alloc_ctx = *caller_alloc_ctx;
- } else {
- util_assume(tsdn != NULL);
- emap_alloc_ctx_lookup(
- tsdn, &arena_emap_global, ptr, &alloc_ctx);
- }
-
- if (config_debug) {
- edata_t *edata = emap_edata_lookup(
- tsdn, &arena_emap_global, ptr);
- assert(alloc_ctx.szind == edata_szind_get(edata));
- assert(alloc_ctx.szind < SC_NSIZES);
- assert(alloc_ctx.slab == edata_slab_get(edata));
- assert(emap_alloc_ctx_usize_get(&alloc_ctx)
- == edata_usize_get(edata));
- }
-
- if (likely(alloc_ctx.slab)) {
- /* Small allocation. */
- if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
- return;
- }
- tcache_dalloc_small(
- tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
- } else {
- arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
- emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
- }
-}
-
-static inline void
-arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
- assert(ptr != NULL);
- assert(size <= SC_LARGE_MAXCLASS);
-
- emap_alloc_ctx_t alloc_ctx;
- if (!config_prof || !opt_prof) {
- /*
- * There is no risk of being confused by a promoted sampled
- * object, so base szind and slab on the given size.
- */
- szind_t szind = sz_size2index(size);
- emap_alloc_ctx_init(
- &alloc_ctx, szind, (szind < SC_NBINS), size);
- }
-
- if ((config_prof && opt_prof) || config_debug) {
- emap_alloc_ctx_lookup(
- tsdn, &arena_emap_global, ptr, &alloc_ctx);
-
- assert(alloc_ctx.szind == sz_size2index(size));
- assert((config_prof && opt_prof)
- || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
-
- if (config_debug) {
- edata_t *edata = emap_edata_lookup(
- tsdn, &arena_emap_global, ptr);
- assert(alloc_ctx.szind == edata_szind_get(edata));
- assert(alloc_ctx.slab == edata_slab_get(edata));
- }
- }
-
- if (likely(alloc_ctx.slab)) {
- /* Small allocation. */
- arena_dalloc_small(tsdn, ptr);
- } else {
- arena_dalloc_large_no_tcache(tsdn, ptr, alloc_ctx.szind,
- emap_alloc_ctx_usize_get(&alloc_ctx));
- }
-}
-
-JEMALLOC_ALWAYS_INLINE void
-arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
- emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
- assert(!tsdn_null(tsdn) || tcache == NULL);
- assert(ptr != NULL);
- assert(size <= SC_LARGE_MAXCLASS);
-
- if (unlikely(tcache == NULL)) {
- arena_sdalloc_no_tcache(tsdn, ptr, size);
- return;
- }
-
- emap_alloc_ctx_t alloc_ctx;
- if (config_prof && opt_prof) {
- if (caller_alloc_ctx == NULL) {
- /* Uncommon case and should be a static check. */
- emap_alloc_ctx_lookup(
- tsdn, &arena_emap_global, ptr, &alloc_ctx);
- assert(alloc_ctx.szind == sz_size2index(size));
- assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
- } else {
- alloc_ctx = *caller_alloc_ctx;
- }
- } else {
- /*
- * There is no risk of being confused by a promoted sampled
- * object, so base szind and slab on the given size.
- */
- alloc_ctx.szind = sz_size2index(size);
- alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
- }
-
- if (config_debug) {
- edata_t *edata = emap_edata_lookup(
- tsdn, &arena_emap_global, ptr);
- assert(alloc_ctx.szind == edata_szind_get(edata));
- assert(alloc_ctx.slab == edata_slab_get(edata));
- emap_alloc_ctx_init(
- &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
- assert(emap_alloc_ctx_usize_get(&alloc_ctx)
- == edata_usize_get(edata));
- }
-
- if (likely(alloc_ctx.slab)) {
- /* Small allocation. */
- if (arena_tcache_dalloc_small_safety_check(tsdn, ptr)) {
- return;
- }
- tcache_dalloc_small(
- tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
- } else {
- arena_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
- sz_s2u(size), slow_path);
- }
-}
-
static inline void
arena_cache_oblivious_randomize(
tsdn_t *tsdn, arena_t *arena, edata_t *edata, size_t alignment) {
diff --git a/include/jemalloc/internal/jemalloc_internal_includes.h b/include/jemalloc/internal/jemalloc_internal_includes.h
index 751c112f..d70c808f 100644
--- a/include/jemalloc/internal/jemalloc_internal_includes.h
+++ b/include/jemalloc/internal/jemalloc_internal_includes.h
@@ -61,6 +61,7 @@
#include "jemalloc/internal/arena_externs.h"
#include "jemalloc/internal/large_externs.h"
#include "jemalloc/internal/tcache_externs.h"
+#include "jemalloc/internal/malloc_dispatch_externs.h"
#include "jemalloc/internal/prof_externs.h"
#include "jemalloc/internal/background_thread_externs.h"
@@ -77,6 +78,7 @@
#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
#include "jemalloc/internal/tcache_inlines.h"
#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/malloc_dispatch_inlines.h"
#include "jemalloc/internal/jemalloc_internal_inlines_c.h"
#include "jemalloc/internal/prof_inlines.h"
#include "jemalloc/internal/background_thread_inlines.h"
diff --git a/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/include/jemalloc/internal/jemalloc_internal_inlines_c.h
index 541821ae..323f32d5 100644
--- a/include/jemalloc/internal/jemalloc_internal_inlines_c.h
+++ b/include/jemalloc/internal/jemalloc_internal_inlines_c.h
@@ -8,6 +8,7 @@
#include "jemalloc/internal/jemalloc_init.h"
#include "jemalloc/internal/jemalloc_internal_types.h"
#include "jemalloc/internal/log.h"
+#include "jemalloc/internal/malloc_dispatch_inlines.h"
#include "jemalloc/internal/sz.h"
#include "jemalloc/internal/thread_event.h"
#include "jemalloc/internal/witness.h"
@@ -67,7 +68,7 @@ iallocztm_explicit_slab(tsdn_t *tsdn, size_t size, szind_t ind, bool zero,
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
}
- ret = arena_malloc(
+ ret = malloc_dispatch_malloc(
tsdn, arena, size, ind, zero, slab, tcache, slow_path);
if (config_stats && is_internal && likely(ret != NULL)) {
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -102,7 +103,8 @@ ipallocztm_explicit_slab(tsdn_t *tsdn, size_t usize, size_t alignment,
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
- ret = arena_palloc(tsdn, arena, usize, alignment, zero, slab, tcache);
+ ret = malloc_dispatch_palloc(
+ tsdn, arena, usize, alignment, zero, slab, tcache);
assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret);
if (config_stats && is_internal && likely(ret != NULL)) {
arena_internal_add(iaalloc(tsdn, ret), isalloc(tsdn, ret));
@@ -156,7 +158,7 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
&& tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) {
assert(tcache == NULL);
}
- arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
+ malloc_dispatch_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path);
}
JEMALLOC_ALWAYS_INLINE void
@@ -169,7 +171,7 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
emap_alloc_ctx_t *alloc_ctx, bool slow_path) {
witness_assert_depth_to_rank(
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0);
- arena_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
+ malloc_dispatch_sdalloc(tsdn, ptr, size, tcache, alloc_ctx, slow_path);
}
JEMALLOC_ALWAYS_INLINE void *
@@ -217,8 +219,8 @@ iralloct_explicit_slab(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size,
zero, slab, tcache, arena);
}
- return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero,
- slab, tcache);
+ return malloc_dispatch_ralloc(
+ tsdn, arena, ptr, oldsize, size, alignment, zero, slab, tcache);
}
JEMALLOC_ALWAYS_INLINE void *
diff --git a/include/jemalloc/internal/malloc_dispatch_externs.h b/include/jemalloc/internal/malloc_dispatch_externs.h
new file mode 100644
index 00000000..197b5d6e
--- /dev/null
+++ b/include/jemalloc/internal/malloc_dispatch_externs.h
@@ -0,0 +1,19 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_EXTERNS_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_EXTERNS_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/tsd_types.h"
+
+/* Forward decls; only used as pointer types below. */
+typedef struct arena_s arena_t;
+typedef struct tcache_s tcache_t;
+
+void *malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+ size_t alignment, bool zero, bool slab, tcache_t *tcache);
+void malloc_dispatch_dalloc_promoted(
+ tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path);
+void *malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr,
+ size_t oldsize, size_t size, size_t alignment, bool zero, bool slab,
+ tcache_t *tcache);
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_EXTERNS_H */
diff --git a/include/jemalloc/internal/malloc_dispatch_inlines.h b/include/jemalloc/internal/malloc_dispatch_inlines.h
new file mode 100644
index 00000000..a3b10c48
--- /dev/null
+++ b/include/jemalloc/internal/malloc_dispatch_inlines.h
@@ -0,0 +1,279 @@
+#ifndef JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+#define JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/arena_externs.h"
+#include "jemalloc/internal/arena_inlines_b.h"
+#include "jemalloc/internal/bin_inlines.h"
+#include "jemalloc/internal/div.h"
+#include "jemalloc/internal/emap.h"
+#include "jemalloc/internal/jemalloc_internal_inlines_b.h"
+#include "jemalloc/internal/jemalloc_internal_types.h"
+#include "jemalloc/internal/large_externs.h"
+#include "jemalloc/internal/malloc_dispatch_externs.h"
+#include "jemalloc/internal/safety_check.h"
+#include "jemalloc/internal/sc.h"
+#include "jemalloc/internal/sz.h"
+#include "jemalloc/internal/tcache_inlines.h"
+
+JEMALLOC_ALWAYS_INLINE void *
+malloc_dispatch_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
+ bool zero, bool slab, tcache_t *tcache, bool slow_path) {
+ assert(!tsdn_null(tsdn) || tcache == NULL);
+
+ if (likely(tcache != NULL)) {
+ if (likely(slab)) {
+ assert(sz_can_use_slab(size));
+ return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache,
+ size, ind, zero, slow_path);
+ } else if (likely(tcache_can_cache_large(tcache, ind))) {
+ return tcache_alloc_large(tsdn_tsd(tsdn), arena, tcache,
+ size, ind, zero, slow_path);
+ }
+ /* (size > tcache_max) case falls through. */
+ }
+
+ return arena_malloc_hard(tsdn, arena, size, ind, zero, slab);
+}
+
+static inline void
+malloc_dispatch_dalloc_large_no_tcache(
+ tsdn_t *tsdn, void *ptr, szind_t szind, size_t usize) {
+ /*
+ * szind is still needed in this function mainly because
+ * szind < SC_NBINS determines not only if this is a small alloc,
+ * but also if szind is valid (an inactive extent would have
+ * szind == SC_NSIZES).
+ */
+ if (config_prof && unlikely(szind < SC_NBINS)) {
+ malloc_dispatch_dalloc_promoted(tsdn, ptr, NULL, true);
+ } else {
+ edata_t *edata = emap_edata_lookup(
+ tsdn, &arena_emap_global, ptr);
+ if (large_dalloc_safety_checks(edata, ptr, usize)) {
+ /* See the comment in isfree. */
+ return;
+ }
+ large_dalloc(tsdn, edata);
+ }
+}
+
+static inline void
+malloc_dispatch_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) {
+ assert(ptr != NULL);
+
+ emap_alloc_ctx_t alloc_ctx;
+ emap_alloc_ctx_lookup(tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+ if (config_debug) {
+ edata_t *edata = emap_edata_lookup(
+ tsdn, &arena_emap_global, ptr);
+ assert(alloc_ctx.szind == edata_szind_get(edata));
+ assert(alloc_ctx.szind < SC_NSIZES);
+ assert(alloc_ctx.slab == edata_slab_get(edata));
+ assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+ == edata_usize_get(edata));
+ }
+
+ if (likely(alloc_ctx.slab)) {
+ /* Small allocation. */
+ arena_dalloc_small(tsdn, ptr);
+ } else {
+ malloc_dispatch_dalloc_large_no_tcache(
+ tsdn, ptr, alloc_ctx.szind,
+ emap_alloc_ctx_usize_get(&alloc_ctx));
+ }
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc_large(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+ szind_t szind, size_t usize, bool slow_path) {
+ assert(!tsdn_null(tsdn) && tcache != NULL);
+ bool is_sample_promoted = config_prof && szind < SC_NBINS;
+ if (unlikely(is_sample_promoted)) {
+ malloc_dispatch_dalloc_promoted(tsdn, ptr, tcache, slow_path);
+ } else {
+ if (tcache_can_cache_large(tcache, szind)) {
+ tcache_dalloc_large(
+ tsdn_tsd(tsdn), tcache, ptr, szind, slow_path);
+ } else {
+ edata_t *edata = emap_edata_lookup(
+ tsdn, &arena_emap_global, ptr);
+ if (large_dalloc_safety_checks(edata, ptr, usize)) {
+ /* See the comment in isfree. */
+ return;
+ }
+ large_dalloc(tsdn, edata);
+ }
+ }
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+malloc_dispatch_dalloc_small_safety_check(tsdn_t *tsdn, void *ptr) {
+ if (!config_debug) {
+ return false;
+ }
+ edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+ szind_t binind = edata_szind_get(edata);
+ div_info_t div_info = arena_binind_div_info[binind];
+ /*
+ * Calls the internal function bin_slab_regind_impl because the
+ * safety check does not require a lock.
+ */
+ size_t regind = bin_slab_regind_impl(&div_info, binind, edata, ptr);
+ slab_data_t *slab_data = edata_slab_data_get(edata);
+ const bin_info_t *bin_info = &bin_infos[binind];
+ assert(edata_nfree_get(edata) < bin_info->nregs);
+ if (unlikely(!bitmap_get(
+ slab_data->bitmap, &bin_info->bitmap_info, regind))) {
+ safety_check_fail(
+ "Invalid deallocation detected: the pointer being freed (%p) not "
+ "currently active, possibly caused by double free bugs.\n",
+ ptr);
+ return true;
+ }
+ return false;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache,
+ emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+ assert(!tsdn_null(tsdn) || tcache == NULL);
+ assert(ptr != NULL);
+
+ if (unlikely(tcache == NULL)) {
+ malloc_dispatch_dalloc_no_tcache(tsdn, ptr);
+ return;
+ }
+
+ emap_alloc_ctx_t alloc_ctx;
+ if (caller_alloc_ctx != NULL) {
+ alloc_ctx = *caller_alloc_ctx;
+ } else {
+ util_assume(tsdn != NULL);
+ emap_alloc_ctx_lookup(
+ tsdn, &arena_emap_global, ptr, &alloc_ctx);
+ }
+
+ if (config_debug) {
+ edata_t *edata = emap_edata_lookup(
+ tsdn, &arena_emap_global, ptr);
+ assert(alloc_ctx.szind == edata_szind_get(edata));
+ assert(alloc_ctx.szind < SC_NSIZES);
+ assert(alloc_ctx.slab == edata_slab_get(edata));
+ assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+ == edata_usize_get(edata));
+ }
+
+ if (likely(alloc_ctx.slab)) {
+ /* Small allocation. */
+ if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+ return;
+ }
+ tcache_dalloc_small(
+ tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+ } else {
+ malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+ emap_alloc_ctx_usize_get(&alloc_ctx), slow_path);
+ }
+}
+
+static inline void
+malloc_dispatch_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) {
+ assert(ptr != NULL);
+ assert(size <= SC_LARGE_MAXCLASS);
+
+ emap_alloc_ctx_t alloc_ctx;
+ if (!config_prof || !opt_prof) {
+ /*
+ * There is no risk of being confused by a promoted sampled
+ * object, so base szind and slab on the given size.
+ */
+ szind_t szind = sz_size2index(size);
+ emap_alloc_ctx_init(
+ &alloc_ctx, szind, (szind < SC_NBINS), size);
+ }
+
+ if ((config_prof && opt_prof) || config_debug) {
+ emap_alloc_ctx_lookup(
+ tsdn, &arena_emap_global, ptr, &alloc_ctx);
+
+ assert(alloc_ctx.szind == sz_size2index(size));
+ assert((config_prof && opt_prof)
+ || alloc_ctx.slab == (alloc_ctx.szind < SC_NBINS));
+
+ if (config_debug) {
+ edata_t *edata = emap_edata_lookup(
+ tsdn, &arena_emap_global, ptr);
+ assert(alloc_ctx.szind == edata_szind_get(edata));
+ assert(alloc_ctx.slab == edata_slab_get(edata));
+ }
+ }
+
+ if (likely(alloc_ctx.slab)) {
+ /* Small allocation. */
+ arena_dalloc_small(tsdn, ptr);
+ } else {
+ malloc_dispatch_dalloc_large_no_tcache(
+ tsdn, ptr, alloc_ctx.szind,
+ emap_alloc_ctx_usize_get(&alloc_ctx));
+ }
+}
+
+JEMALLOC_ALWAYS_INLINE void
+malloc_dispatch_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache,
+ emap_alloc_ctx_t *caller_alloc_ctx, bool slow_path) {
+ assert(!tsdn_null(tsdn) || tcache == NULL);
+ assert(ptr != NULL);
+ assert(size <= SC_LARGE_MAXCLASS);
+
+ if (unlikely(tcache == NULL)) {
+ malloc_dispatch_sdalloc_no_tcache(tsdn, ptr, size);
+ return;
+ }
+
+ emap_alloc_ctx_t alloc_ctx;
+ if (config_prof && opt_prof) {
+ if (caller_alloc_ctx == NULL) {
+ /* Uncommon case and should be a static check. */
+ emap_alloc_ctx_lookup(
+ tsdn, &arena_emap_global, ptr, &alloc_ctx);
+ assert(alloc_ctx.szind == sz_size2index(size));
+ assert(emap_alloc_ctx_usize_get(&alloc_ctx) == size);
+ } else {
+ alloc_ctx = *caller_alloc_ctx;
+ }
+ } else {
+ /*
+ * There is no risk of being confused by a promoted sampled
+ * object, so base szind and slab on the given size.
+ */
+ alloc_ctx.szind = sz_size2index(size);
+ alloc_ctx.slab = (alloc_ctx.szind < SC_NBINS);
+ }
+
+ if (config_debug) {
+ edata_t *edata = emap_edata_lookup(
+ tsdn, &arena_emap_global, ptr);
+ assert(alloc_ctx.szind == edata_szind_get(edata));
+ assert(alloc_ctx.slab == edata_slab_get(edata));
+ emap_alloc_ctx_init(
+ &alloc_ctx, alloc_ctx.szind, alloc_ctx.slab, sz_s2u(size));
+ assert(emap_alloc_ctx_usize_get(&alloc_ctx)
+ == edata_usize_get(edata));
+ }
+
+ if (likely(alloc_ctx.slab)) {
+ /* Small allocation. */
+ if (malloc_dispatch_dalloc_small_safety_check(tsdn, ptr)) {
+ return;
+ }
+ tcache_dalloc_small(
+ tsdn_tsd(tsdn), tcache, ptr, alloc_ctx.szind, slow_path);
+ } else {
+ malloc_dispatch_dalloc_large(tsdn, ptr, tcache, alloc_ctx.szind,
+ sz_s2u(size), slow_path);
+ }
+}
+
+#endif /* JEMALLOC_INTERNAL_MALLOC_DISPATCH_INLINES_H */
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index 1ba81aad..a48ca889 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -73,6 +73,7 @@
+
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
index c196ce59..e4bbe65a 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
@@ -94,6 +94,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 62c36ea5..bc2685c0 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -73,6 +73,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
index c196ce59..e4bbe65a 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
@@ -94,6 +94,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
index ed35784b..dffda081 100644
--- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
@@ -73,6 +73,7 @@
+
diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
index c196ce59..e4bbe65a 100644
--- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
@@ -94,6 +94,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
index 7c84196d..c48f9a7b 100644
--- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
@@ -73,6 +73,7 @@
+
diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
index c196ce59..e4bbe65a 100644
--- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
@@ -94,6 +94,9 @@
Source Files
+
+ Source Files
+
Source Files
diff --git a/src/arena.c b/src/arena.c
index d8bd7ae7..8f8b11fb 100644
--- a/src/arena.c
+++ b/src/arena.c
@@ -694,11 +694,13 @@ arena_prof_promote(tsdn_t *tsdn, void *ptr, size_t usize, size_t bumped_usize) {
assert(isalloc(tsdn, ptr) == usize);
}
-static size_t
+size_t
arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
cassert(config_prof);
+ assert(opt_prof);
assert(ptr != NULL);
- size_t usize = isalloc(tsdn, ptr);
+ size_t usize = edata_usize_get(edata);
+ assert(isalloc(tsdn, ptr) == usize);
size_t bumped_usize = sz_sa2u(usize, PROF_SAMPLE_ALIGNMENT);
assert(bumped_usize <= SC_LARGE_MINCLASS
&& PAGE_CEILING(bumped_usize) == bumped_usize);
@@ -710,17 +712,6 @@ arena_prof_demote(tsdn_t *tsdn, edata_t *edata, const void *ptr) {
assert(isalloc(tsdn, ptr) == bumped_usize);
- return bumped_usize;
-}
-
-static void
-arena_dalloc_promoted_impl(
- tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path, edata_t *edata) {
- cassert(config_prof);
- assert(opt_prof);
-
- size_t usize = edata_usize_get(edata);
- size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
if (config_opt_safety_checks && usize < SC_LARGE_MINCLASS) {
/*
* Currently, we only do redzoning for small sampled
@@ -728,21 +719,8 @@ arena_dalloc_promoted_impl(
*/
safety_check_verify_redzone(ptr, usize, bumped_usize);
}
- szind_t bumped_ind = sz_size2index(bumped_usize);
- if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
- && tcache_can_cache_large(tcache, bumped_ind)) {
- tcache_dalloc_large(
- tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
- } else {
- large_dalloc(tsdn, edata);
- }
-}
-void
-arena_dalloc_promoted(
- tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
- edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
- arena_dalloc_promoted_impl(tsdn, ptr, tcache, slow_path, edata);
+ return bumped_usize;
}
void
@@ -784,8 +762,8 @@ arena_reset(tsd_t *tsd, arena_t *arena) {
prof_free(tsd, ptr, usize, &alloc_ctx);
}
if (config_prof && opt_prof && alloc_ctx.szind < SC_NBINS) {
- arena_dalloc_promoted_impl(tsd_tsdn(tsd), ptr,
- /* tcache */ NULL, /* slow_path */ true, edata);
+ arena_prof_demote(tsd_tsdn(tsd), edata, ptr);
+ large_dalloc(tsd_tsdn(tsd), edata);
} else {
large_dalloc(tsd_tsdn(tsd), edata);
}
@@ -1154,33 +1132,6 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind,
}
}
-void *
-arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment,
- bool zero, bool slab, tcache_t *tcache) {
- if (slab) {
- assert(sz_can_use_slab(usize));
- /* Small; alignment doesn't require special slab placement. */
-
- /* usize should be a result of sz_sa2u() */
- assert((usize & (alignment - 1)) == 0);
-
- /*
- * Small usize can't come from an alignment larger than a page.
- */
- assert(alignment <= PAGE);
-
- return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
- zero, slab, tcache, true);
- } else {
- if (likely(alignment <= CACHELINE)) {
- return large_malloc(tsdn, arena, usize, zero);
- } else {
- return large_palloc(
- tsdn, arena, usize, alignment, zero);
- }
- }
-}
-
static void
arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, edata_t *edata, void *ptr) {
szind_t binind = edata_szind_get(edata);
@@ -1607,64 +1558,6 @@ done:
return ret;
}
-static void *
-arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
- size_t alignment, bool zero, bool slab, tcache_t *tcache) {
- if (alignment == 0) {
- return arena_malloc(tsdn, arena, usize, sz_size2index(usize),
- zero, slab, tcache, true);
- }
- usize = sz_sa2u(usize, alignment);
- if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
- return NULL;
- }
- return ipalloct_explicit_slab(
- tsdn, usize, alignment, zero, slab, tcache, arena);
-}
-
-void *
-arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
- size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
- size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
- if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
- return NULL;
- }
-
- if (likely(slab)) {
- assert(sz_can_use_slab(usize));
- /* Try to avoid moving the allocation. */
- UNUSED size_t newsize;
- if (!arena_ralloc_no_move(
- tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
- return ptr;
- }
- }
-
- if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
- return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
- tcache);
- }
-
- /*
- * size and oldsize are different enough that we need to move the
- * object. In that case, fall back to allocating new space and copying.
- */
- void *ret = arena_ralloc_move_helper(
- tsdn, arena, usize, alignment, zero, slab, tcache);
- if (ret == NULL) {
- return NULL;
- }
-
- /*
- * Junk/zero-filling were already done by
- * ipalloc()/arena_malloc().
- */
- size_t copysize = (usize < oldsize) ? usize : oldsize;
- memcpy(ret, ptr, copysize);
- isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
- return ret;
-}
-
ehooks_t *
arena_get_ehooks(const arena_t *arena) {
return base_ehooks_get(arena->base);
diff --git a/src/malloc_dispatch.c b/src/malloc_dispatch.c
new file mode 100644
index 00000000..ea8d2817
--- /dev/null
+++ b/src/malloc_dispatch.c
@@ -0,0 +1,109 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/emap.h"
+
+/******************************************************************************/
+
+void
+malloc_dispatch_dalloc_promoted(
+ tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
+ cassert(config_prof);
+ assert(opt_prof);
+
+ edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+ size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
+ szind_t bumped_ind = sz_size2index(bumped_usize);
+ if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
+ && tcache_can_cache_large(tcache, bumped_ind)) {
+ tcache_dalloc_large(
+ tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
+ } else {
+ large_dalloc(tsdn, edata);
+ }
+}
+
+void *
+malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+ size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+ if (slab) {
+ assert(sz_can_use_slab(usize));
+ /* Small; alignment doesn't require special slab placement. */
+
+ /* usize should be a result of sz_sa2u() */
+ assert((usize & (alignment - 1)) == 0);
+
+ /*
+ * Small usize can't come from an alignment larger than a page.
+ */
+ assert(alignment <= PAGE);
+
+ return malloc_dispatch_malloc(tsdn, arena, usize,
+ sz_size2index(usize), zero, slab, tcache, true);
+ } else {
+ if (likely(alignment <= CACHELINE)) {
+ return large_malloc(tsdn, arena, usize, zero);
+ } else {
+ return large_palloc(
+ tsdn, arena, usize, alignment, zero);
+ }
+ }
+}
+
+static void *
+malloc_dispatch_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
+ size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+ if (alignment == 0) {
+ return malloc_dispatch_malloc(tsdn, arena, usize,
+ sz_size2index(usize), zero, slab, tcache, true);
+ }
+ usize = sz_sa2u(usize, alignment);
+ if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
+ return NULL;
+ }
+ return ipalloct_explicit_slab(
+ tsdn, usize, alignment, zero, slab, tcache, arena);
+}
+
+void *
+malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
+ size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+ size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
+ if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
+ return NULL;
+ }
+
+ if (likely(slab)) {
+ assert(sz_can_use_slab(usize));
+ /* Try to avoid moving the allocation. */
+ UNUSED size_t newsize;
+ if (!arena_ralloc_no_move(
+ tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
+ return ptr;
+ }
+ }
+
+ if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
+ return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
+ tcache);
+ }
+
+ /*
+ * size and oldsize are different enough that we need to move the
+ * object. In that case, fall back to allocating new space and copying.
+ */
+ void *ret = malloc_dispatch_ralloc_move_helper(
+ tsdn, arena, usize, alignment, zero, slab, tcache);
+ if (ret == NULL) {
+ return NULL;
+ }
+
+ /*
+ * Junk/zero-filling were already done by ipalloc() / dispatch alloc.
+ */
+ size_t copysize = (usize < oldsize) ? usize : oldsize;
+ memcpy(ret, ptr, copysize);
+ isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
+ return ret;
+}