diff --git a/Makefile.in b/Makefile.in
index 4dd4ce85..7365a923 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -123,6 +123,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
$(srcroot)src/san_bump.c \
$(srcroot)src/hook.c \
$(srcroot)src/hpa.c \
+ $(srcroot)src/hpa_central.c \
$(srcroot)src/hpa_hooks.c \
$(srcroot)src/hpa_utils.c \
$(srcroot)src/hpdata.c \
diff --git a/include/jemalloc/internal/hpa.h b/include/jemalloc/internal/hpa.h
index 131bbb90..06567740 100644
--- a/include/jemalloc/internal/hpa.h
+++ b/include/jemalloc/internal/hpa.h
@@ -6,36 +6,13 @@
#include "jemalloc/internal/edata_cache.h"
#include "jemalloc/internal/emap.h"
#include "jemalloc/internal/exp_grow.h"
+#include "jemalloc/internal/hpa_central.h"
#include "jemalloc/internal/hpa_hooks.h"
#include "jemalloc/internal/hpa_opts.h"
#include "jemalloc/internal/mutex.h"
#include "jemalloc/internal/pai.h"
#include "jemalloc/internal/psset.h"
-typedef struct hpa_central_s hpa_central_t;
-struct hpa_central_s {
- /*
- * Guards expansion of eden. We separate this from the regular mutex so
- * that cheaper operations can still continue while we're doing the OS
- * call.
- */
- malloc_mutex_t grow_mtx;
- /*
- * Either NULL (if empty), or some integer multiple of a
- * hugepage-aligned number of hugepages. We carve them off one at a
- * time to satisfy new pageslab requests.
- *
- * Guarded by grow_mtx.
- */
- void *eden;
- size_t eden_len;
- /* Source for metadata. */
- base_t *base;
-
- /* The HPA hooks. */
- hpa_hooks_t hooks;
-};
-
typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
struct hpa_shard_nonderived_stats_s {
/*
@@ -165,8 +142,6 @@ bool hpa_hugepage_size_exceeds_limit(void);
* just that it can function properly given the system it's running on.
*/
bool hpa_supported(void);
-bool hpa_central_init(
- hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
base_t *base, edata_cache_t *edata_cache, unsigned ind,
const hpa_shard_opts_t *opts);
diff --git a/include/jemalloc/internal/hpa_central.h b/include/jemalloc/internal/hpa_central.h
new file mode 100644
index 00000000..3e0ff7da
--- /dev/null
+++ b/include/jemalloc/internal/hpa_central.h
@@ -0,0 +1,41 @@
+#ifndef JEMALLOC_INTERNAL_HPA_CENTRAL_H
+#define JEMALLOC_INTERNAL_HPA_CENTRAL_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/base.h"
+#include "jemalloc/internal/hpa_hooks.h"
+#include "jemalloc/internal/hpdata.h"
+#include "jemalloc/internal/mutex.h"
+#include "jemalloc/internal/tsd_types.h"
+
+typedef struct hpa_central_s hpa_central_t;
+struct hpa_central_s {
+ /*
+ * Guards expansion of eden. We separate this from the regular mutex so
+ * that cheaper operations can still continue while we're doing the OS
+ * call.
+ */
+ malloc_mutex_t grow_mtx;
+ /*
+ * Either NULL (if empty), or some integer multiple of a
+ * hugepage-aligned number of hugepages. We carve them off one at a
+ * time to satisfy new pageslab requests.
+ *
+ * Guarded by grow_mtx.
+ */
+ void *eden;
+ size_t eden_len;
+ /* Source for metadata. */
+ base_t *base;
+
+ /* The HPA hooks. */
+ hpa_hooks_t hooks;
+};
+
+bool hpa_central_init(
+ hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
+
+hpdata_t *hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
+ uint64_t age, bool hugify_eager, bool *oom);
+
+#endif /* JEMALLOC_INTERNAL_HPA_CENTRAL_H */
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
index abdeb7b7..bfb62d78 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj
@@ -61,6 +61,7 @@
+
diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
index 7ce66945..26408c8e 100644
--- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters
@@ -67,6 +67,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -206,4 +209,4 @@
Source Files
-
\ No newline at end of file
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
index 1f39cb91..037eb724 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj
@@ -61,6 +61,7 @@
+
diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
index 7ce66945..26408c8e 100644
--- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters
@@ -67,6 +67,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -206,4 +209,4 @@
Source Files
-
\ No newline at end of file
+
diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
index 0b1e1707..bd6595b1 100644
--- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj
@@ -61,6 +61,7 @@
+
diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
index 7ce66945..26408c8e 100644
--- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters
@@ -67,6 +67,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -206,4 +209,4 @@
Source Files
-
\ No newline at end of file
+
diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
index 54462516..3f880176 100644
--- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
+++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj
@@ -61,6 +61,7 @@
+
diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
index 7ce66945..26408c8e 100644
--- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
+++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters
@@ -67,6 +67,9 @@
Source Files
+
+ Source Files
+
Source Files
@@ -206,4 +209,4 @@
Source Files
-
\ No newline at end of file
+
diff --git a/src/hpa.c b/src/hpa.c
index 5e3727a1..cc330379 100644
--- a/src/hpa.c
+++ b/src/hpa.c
@@ -8,8 +8,6 @@
#include "jemalloc/internal/witness.h"
#include "jemalloc/internal/jemalloc_probe.h"
-#define HPA_EDEN_SIZE (128 * HUGEPAGE)
-
static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
bool *deferred_work_generated);
@@ -75,119 +73,6 @@ hpa_do_consistency_checks(hpa_shard_t *shard) {
assert(shard->base != NULL);
}
-bool
-hpa_central_init(
- hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
- /* malloc_conf processing should have filtered out these cases. */
- assert(hpa_supported());
- bool err;
- err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow",
- WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
- if (err) {
- return true;
- }
-
- central->base = base;
- central->eden = NULL;
- central->eden_len = 0;
- central->hooks = *hooks;
- return false;
-}
-
-static hpdata_t *
-hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
- return (hpdata_t *)base_alloc(
- tsdn, central->base, sizeof(hpdata_t), CACHELINE);
-}
-
-static hpdata_t *
-hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
- uint64_t age, bool hugify_eager, bool *oom) {
- /* Don't yet support big allocations; these should get filtered out. */
- assert(size <= HUGEPAGE);
- /*
- * Should only try to extract from the central allocator if the local
- * shard is exhausted. We should hold the grow_mtx on that shard.
- */
- witness_assert_positive_depth_to_rank(
- tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
-
- malloc_mutex_lock(tsdn, ¢ral->grow_mtx);
- *oom = false;
-
- hpdata_t *ps = NULL;
- bool start_as_huge = hugify_eager
- || (init_system_thp_mode == system_thp_mode_always
- && opt_experimental_hpa_start_huge_if_thp_always);
-
- /* Is eden a perfect fit? */
- if (central->eden != NULL && central->eden_len == HUGEPAGE) {
- ps = hpa_alloc_ps(tsdn, central);
- if (ps == NULL) {
- *oom = true;
- malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
- return NULL;
- }
- hpdata_init(ps, central->eden, age, start_as_huge);
- central->eden = NULL;
- central->eden_len = 0;
- malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
- return ps;
- }
-
- /*
- * We're about to try to allocate from eden by splitting. If eden is
- * NULL, we have to allocate it too. Otherwise, we just have to
- * allocate an edata_t for the new psset.
- */
- if (central->eden == NULL) {
- /* Allocate address space, bailing if we fail. */
- void *new_eden = central->hooks.map(HPA_EDEN_SIZE);
- if (new_eden == NULL) {
- *oom = true;
- malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
- return NULL;
- }
- if (hugify_eager) {
- central->hooks.hugify(
- new_eden, HPA_EDEN_SIZE, /* sync */ false);
- }
- ps = hpa_alloc_ps(tsdn, central);
- if (ps == NULL) {
- central->hooks.unmap(new_eden, HPA_EDEN_SIZE);
- *oom = true;
- malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
- return NULL;
- }
- central->eden = new_eden;
- central->eden_len = HPA_EDEN_SIZE;
- } else {
- /* Eden is already nonempty; only need an edata for ps. */
- ps = hpa_alloc_ps(tsdn, central);
- if (ps == NULL) {
- *oom = true;
- malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
- return NULL;
- }
- }
- assert(ps != NULL);
- assert(central->eden != NULL);
- assert(central->eden_len > HUGEPAGE);
- assert(central->eden_len % HUGEPAGE == 0);
- assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
-
- hpdata_init(ps, central->eden, age, start_as_huge);
-
- char *eden_char = (char *)central->eden;
- eden_char += HUGEPAGE;
- central->eden = (void *)eden_char;
- central->eden_len -= HUGEPAGE;
-
- malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
-
- return ps;
-}
-
bool
hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
base_t *base, edata_cache_t *edata_cache, unsigned ind,
diff --git a/src/hpa_central.c b/src/hpa_central.c
new file mode 100644
index 00000000..b4f770c2
--- /dev/null
+++ b/src/hpa_central.c
@@ -0,0 +1,121 @@
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/jemalloc_internal_includes.h"
+
+#include "jemalloc/internal/hpa_central.h"
+#include "jemalloc/internal/tsd.h"
+#include "jemalloc/internal/witness.h"
+
+#define HPA_EDEN_SIZE (128 * HUGEPAGE)
+
+bool
+hpa_central_init(
+ hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
+ /* malloc_conf processing should have filtered out these cases. */
+ assert(hpa_supported());
+ bool err;
+ err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow",
+ WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
+ if (err) {
+ return true;
+ }
+
+ central->base = base;
+ central->eden = NULL;
+ central->eden_len = 0;
+ central->hooks = *hooks;
+ return false;
+}
+
+static hpdata_t *
+hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
+ return (hpdata_t *)base_alloc(
+ tsdn, central->base, sizeof(hpdata_t), CACHELINE);
+}
+
+hpdata_t *
+hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
+ uint64_t age, bool hugify_eager, bool *oom) {
+ /* Don't yet support big allocations; these should get filtered out. */
+ assert(size <= HUGEPAGE);
+ /*
+ * Should only try to extract from the central allocator if the local
+ * shard is exhausted. We should hold the grow_mtx on that shard.
+ */
+ witness_assert_positive_depth_to_rank(
+ tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
+
+ malloc_mutex_lock(tsdn, ¢ral->grow_mtx);
+ *oom = false;
+
+ hpdata_t *ps = NULL;
+ bool start_as_huge = hugify_eager
+ || (init_system_thp_mode == system_thp_mode_always
+ && opt_experimental_hpa_start_huge_if_thp_always);
+
+ /* Is eden a perfect fit? */
+ if (central->eden != NULL && central->eden_len == HUGEPAGE) {
+ ps = hpa_alloc_ps(tsdn, central);
+ if (ps == NULL) {
+ *oom = true;
+ malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
+ return NULL;
+ }
+ hpdata_init(ps, central->eden, age, start_as_huge);
+ central->eden = NULL;
+ central->eden_len = 0;
+ malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
+ return ps;
+ }
+
+ /*
+ * We're about to try to allocate from eden by splitting. If eden is
+ * NULL, we have to allocate it too. Otherwise, we just have to
+ * allocate an edata_t for the new psset.
+ */
+ if (central->eden == NULL) {
+ /* Allocate address space, bailing if we fail. */
+ void *new_eden = central->hooks.map(HPA_EDEN_SIZE);
+ if (new_eden == NULL) {
+ *oom = true;
+ malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
+ return NULL;
+ }
+ if (hugify_eager) {
+ central->hooks.hugify(
+ new_eden, HPA_EDEN_SIZE, /* sync */ false);
+ }
+ ps = hpa_alloc_ps(tsdn, central);
+ if (ps == NULL) {
+ central->hooks.unmap(new_eden, HPA_EDEN_SIZE);
+ *oom = true;
+ malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
+ return NULL;
+ }
+ central->eden = new_eden;
+ central->eden_len = HPA_EDEN_SIZE;
+ } else {
+ /* Eden is already nonempty; only need an edata for ps. */
+ ps = hpa_alloc_ps(tsdn, central);
+ if (ps == NULL) {
+ *oom = true;
+ malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
+ return NULL;
+ }
+ }
+ assert(ps != NULL);
+ assert(central->eden != NULL);
+ assert(central->eden_len > HUGEPAGE);
+ assert(central->eden_len % HUGEPAGE == 0);
+ assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
+
+ hpdata_init(ps, central->eden, age, start_as_huge);
+
+ char *eden_char = (char *)central->eden;
+ eden_char += HUGEPAGE;
+ central->eden = (void *)eden_char;
+ central->eden_len -= HUGEPAGE;
+
+ malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
+
+ return ps;
+}