Move malloc routing into new malloc_dispatch module

Pull the tcache-aware allocation routing helpers out of arena into a
layer that sits directly below the public malloc interface:

  arena_malloc          -> malloc_dispatch_malloc
  arena_palloc          -> malloc_dispatch_palloc
  arena_ralloc          -> malloc_dispatch_ralloc
  arena_dalloc*         -> malloc_dispatch_dalloc*
  arena_sdalloc*        -> malloc_dispatch_sdalloc*
  arena_dalloc_promoted -> malloc_dispatch_dalloc_promoted

These helpers decide whether to route through tcache or fall through to
arena/large fast paths.  They are now owned by malloc_dispatch_inlines.h
+ src/malloc_dispatch.c, and the only consumers are the public-front-end
wrappers in jemalloc_internal_inlines_c.h.

arena keeps a narrower arena_prof_demote() helper for the sampled
allocation demotion + redzone verification it used to perform inline.
arena_inlines_b.h no longer includes tcache_inlines.h -- the symbol
level arena <-> tcache cycle is gone (it's now in malloc_dispatch).

src/malloc_dispatch.c

@@ -0,0 +1,109 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/assert.h"
+#include "jemalloc/internal/emap.h"
+
+/******************************************************************************/
+
+void
+malloc_dispatch_dalloc_promoted(
+    tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path) {
+	cassert(config_prof);
+	assert(opt_prof);
+
+	edata_t *edata = emap_edata_lookup(tsdn, &arena_emap_global, ptr);
+	size_t bumped_usize = arena_prof_demote(tsdn, edata, ptr);
+	szind_t bumped_ind = sz_size2index(bumped_usize);
+	if (bumped_usize >= SC_LARGE_MINCLASS && tcache != NULL
+	    && tcache_can_cache_large(tcache, bumped_ind)) {
+		tcache_dalloc_large(
+		    tsdn_tsd(tsdn), tcache, ptr, bumped_ind, slow_path);
+	} else {
+		large_dalloc(tsdn, edata);
+	}
+}
+
+void *
+malloc_dispatch_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+	if (slab) {
+		assert(sz_can_use_slab(usize));
+		/* Small; alignment doesn't require special slab placement. */
+
+		/* usize should be a result of sz_sa2u() */
+		assert((usize & (alignment - 1)) == 0);
+
+		/*
+		 * Small usize can't come from an alignment larger than a page.
+		 */
+		assert(alignment <= PAGE);
+
+		return malloc_dispatch_malloc(tsdn, arena, usize,
+		    sz_size2index(usize), zero, slab, tcache, true);
+	} else {
+		if (likely(alignment <= CACHELINE)) {
+			return large_malloc(tsdn, arena, usize, zero);
+		} else {
+			return large_palloc(
+			    tsdn, arena, usize, alignment, zero);
+		}
+	}
+}
+
+static void *
+malloc_dispatch_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize,
+    size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+	if (alignment == 0) {
+		return malloc_dispatch_malloc(tsdn, arena, usize,
+		    sz_size2index(usize), zero, slab, tcache, true);
+	}
+	usize = sz_sa2u(usize, alignment);
+	if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) {
+		return NULL;
+	}
+	return ipalloct_explicit_slab(
+	    tsdn, usize, alignment, zero, slab, tcache, arena);
+}
+
+void *
+malloc_dispatch_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize,
+    size_t size, size_t alignment, bool zero, bool slab, tcache_t *tcache) {
+	size_t usize = alignment == 0 ? sz_s2u(size) : sz_sa2u(size, alignment);
+	if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) {
+		return NULL;
+	}
+
+	if (likely(slab)) {
+		assert(sz_can_use_slab(usize));
+		/* Try to avoid moving the allocation. */
+		UNUSED size_t newsize;
+		if (!arena_ralloc_no_move(
+		        tsdn, ptr, oldsize, usize, 0, zero, &newsize)) {
+			return ptr;
+		}
+	}
+
+	if (oldsize >= SC_LARGE_MINCLASS && usize >= SC_LARGE_MINCLASS) {
+		return large_ralloc(tsdn, arena, ptr, usize, alignment, zero,
+		    tcache);
+	}
+
+	/*
+	 * size and oldsize are different enough that we need to move the
+	 * object.  In that case, fall back to allocating new space and copying.
+	 */
+	void *ret = malloc_dispatch_ralloc_move_helper(
+	    tsdn, arena, usize, alignment, zero, slab, tcache);
+	if (ret == NULL) {
+		return NULL;
+	}
+
+	/*
+	 * Junk/zero-filling were already done by ipalloc() / dispatch alloc.
+	 */
+	size_t copysize = (usize < oldsize) ? usize : oldsize;
+	memcpy(ret, ptr, copysize);
+	isdalloct(tsdn, ptr, oldsize, tcache, NULL, true);
+	return ret;
+}