From ba1e2fe4db78911a56d254e1bd69e17233dde61b Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Fri, 24 Apr 2026 11:06:50 -0700 Subject: [PATCH] Extract fork orchestration into jemalloc_fork module --- Makefile.in | 1 + include/jemalloc/internal/jemalloc_fork.h | 8 + .../internal/jemalloc_internal_externs.h | 3 - .../projects/vc2015/jemalloc/jemalloc.vcxproj | 1 + .../vc2015/jemalloc/jemalloc.vcxproj.filters | 3 + .../projects/vc2017/jemalloc/jemalloc.vcxproj | 1 + .../vc2017/jemalloc/jemalloc.vcxproj.filters | 3 + .../projects/vc2019/jemalloc/jemalloc.vcxproj | 1 + .../vc2019/jemalloc/jemalloc.vcxproj.filters | 3 + .../projects/vc2022/jemalloc/jemalloc.vcxproj | 1 + .../vc2022/jemalloc/jemalloc.vcxproj.filters | 3 + src/jemalloc.c | 177 ------------------ src/jemalloc_fork.c | 163 ++++++++++++++++ src/jemalloc_init.c | 22 +++ src/zone.c | 1 + 15 files changed, 211 insertions(+), 180 deletions(-) create mode 100644 include/jemalloc/internal/jemalloc_fork.h create mode 100644 src/jemalloc_fork.c diff --git a/Makefile.in b/Makefile.in index a8d5ff5e..d6b36971 100644 --- a/Makefile.in +++ b/Makefile.in @@ -95,6 +95,7 @@ LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix) BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h C_SRCS := $(srcroot)src/jemalloc.c \ + $(srcroot)src/jemalloc_fork.c \ $(srcroot)src/jemalloc_init.c \ $(srcroot)src/arena.c \ $(srcroot)src/arenas_management.c \ diff --git a/include/jemalloc/internal/jemalloc_fork.h b/include/jemalloc/internal/jemalloc_fork.h new file mode 100644 index 00000000..3fd8c5c0 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_fork.h @@ -0,0 +1,8 @@ +#ifndef JEMALLOC_INTERNAL_JEMALLOC_FORK_H +#define JEMALLOC_INTERNAL_JEMALLOC_FORK_H + +void jemalloc_prefork(void); +void jemalloc_postfork_parent(void); +void jemalloc_postfork_child(void); + +#endif /* JEMALLOC_INTERNAL_JEMALLOC_FORK_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_externs.h b/include/jemalloc/internal/jemalloc_internal_externs.h index 43057b1a..bbbd044f 100644 --- a/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/include/jemalloc/internal/jemalloc_internal_externs.h @@ -57,9 +57,6 @@ void *bootstrap_malloc(size_t size); void *bootstrap_calloc(size_t num, size_t size); void bootstrap_free(void *ptr); size_t batch_alloc(void **ptrs, size_t num, size_t size, int flags); -void jemalloc_prefork(void); -void jemalloc_postfork_parent(void); -void jemalloc_postfork_child(void); void sdallocx_default(void *ptr, size_t size, int flags); void free_default(void *ptr); void *malloc_default(size_t size); diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index 881e1862..1ba81aad 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -69,6 +69,7 @@ + diff --git a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index 7595606f..c196ce59 100644 --- a/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -208,6 +208,9 @@ Source Files + + Source Files + Source Files diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj index b655de65..62c36ea5 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -69,6 +69,7 @@ + diff --git a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters index 7595606f..c196ce59 100644 --- a/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -208,6 +208,9 @@ Source Files + + Source Files + Source Files diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj index 790d79d8..ed35784b 100644 --- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj @@ -69,6 +69,7 @@ + diff --git a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters index 7595606f..c196ce59 100644 --- a/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2019/jemalloc/jemalloc.vcxproj.filters @@ -208,6 +208,9 @@ Source Files + + Source Files + Source Files diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj index 9dfc7d84..7c84196d 100644 --- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj +++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj @@ -69,6 +69,7 @@ + diff --git a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters index 7595606f..c196ce59 100644 --- a/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters +++ b/msvc/projects/vc2022/jemalloc/jemalloc.vcxproj.filters @@ -208,6 +208,9 @@ Source Files + + Source Files + Source Files diff --git a/src/jemalloc.c b/src/jemalloc.c index 76835068..a3694761 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2273,180 +2273,3 @@ label_done: * End non-standard functions. */ /******************************************************************************/ -/* - * The following functions are used by threading libraries for protection of - * malloc during fork(). - */ - -/* - * If an application creates a thread before doing any allocation in the main - * thread, then calls fork(2) in the main thread followed by memory allocation - * in the child process, a race can occur that results in deadlock within the - * child: the main thread may have forked while the created thread had - * partially initialized the allocator. Ordinarily jemalloc prevents - * fork/malloc races via the following functions it registers during - * initialization using pthread_atfork(), but of course that does no good if - * the allocator isn't fully initialized at fork time. The following library - * constructor is a partial solution to this problem. It may still be possible - * to trigger the deadlock described above, but doing so would involve forking - * via a library constructor that runs before jemalloc's runs. - */ -#ifndef JEMALLOC_JET -JEMALLOC_ATTR(constructor) -static void -jemalloc_constructor(void) { - malloc_init(); -} -#endif - -#ifndef JEMALLOC_MUTEX_INIT_CB -void -jemalloc_prefork(void) -#else -JEMALLOC_EXPORT void -_malloc_prefork(void) -#endif -{ - tsd_t *tsd; - unsigned i, j, narenas; - arena_t *arena; - -#ifdef JEMALLOC_MUTEX_INIT_CB - if (!malloc_initialized()) { - return; - } -#endif - assert(malloc_initialized()); - - tsd = tsd_fetch(); - - narenas = narenas_total_get(); - - witness_prefork(tsd_witness_tsdp_get(tsd)); - /* Acquire all mutexes in a safe order. */ - ctl_prefork(tsd_tsdn(tsd)); - tcache_prefork(tsd_tsdn(tsd)); - arenas_management_prefork(tsd_tsdn(tsd)); - if (have_background_thread) { - background_thread_prefork0(tsd_tsdn(tsd)); - } - prof_prefork0(tsd_tsdn(tsd)); - if (have_background_thread) { - background_thread_prefork1(tsd_tsdn(tsd)); - } - /* Break arena prefork into stages to preserve lock order. */ - for (i = 0; i < 9; i++) { - for (j = 0; j < narenas; j++) { - if ((arena = arena_get(tsd_tsdn(tsd), j, false)) - != NULL) { - switch (i) { - case 0: - arena_prefork0(tsd_tsdn(tsd), arena); - break; - case 1: - arena_prefork1(tsd_tsdn(tsd), arena); - break; - case 2: - arena_prefork2(tsd_tsdn(tsd), arena); - break; - case 3: - arena_prefork3(tsd_tsdn(tsd), arena); - break; - case 4: - arena_prefork4(tsd_tsdn(tsd), arena); - break; - case 5: - arena_prefork5(tsd_tsdn(tsd), arena); - break; - case 6: - arena_prefork6(tsd_tsdn(tsd), arena); - break; - case 7: - arena_prefork7(tsd_tsdn(tsd), arena); - break; - case 8: - arena_prefork8(tsd_tsdn(tsd), arena); - break; - default: - not_reached(); - } - } - } - } - prof_prefork1(tsd_tsdn(tsd)); - stats_prefork(tsd_tsdn(tsd)); - tsd_prefork(tsd); -} - -#ifndef JEMALLOC_MUTEX_INIT_CB -void -jemalloc_postfork_parent(void) -#else -JEMALLOC_EXPORT void -_malloc_postfork(void) -#endif -{ - tsd_t *tsd; - unsigned i, narenas; - -#ifdef JEMALLOC_MUTEX_INIT_CB - if (!malloc_initialized()) { - return; - } -#endif - assert(malloc_initialized()); - - tsd = tsd_fetch(); - - tsd_postfork_parent(tsd); - - witness_postfork_parent(tsd_witness_tsdp_get(tsd)); - /* Release all mutexes, now that fork() has completed. */ - stats_postfork_parent(tsd_tsdn(tsd)); - for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { - arena_t *arena; - - if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { - arena_postfork_parent(tsd_tsdn(tsd), arena); - } - } - prof_postfork_parent(tsd_tsdn(tsd)); - if (have_background_thread) { - background_thread_postfork_parent(tsd_tsdn(tsd)); - } - arenas_management_postfork_parent(tsd_tsdn(tsd)); - tcache_postfork_parent(tsd_tsdn(tsd)); - ctl_postfork_parent(tsd_tsdn(tsd)); -} - -void -jemalloc_postfork_child(void) { - tsd_t *tsd; - unsigned i, narenas; - - assert(malloc_initialized()); - - tsd = tsd_fetch(); - - tsd_postfork_child(tsd); - - witness_postfork_child(tsd_witness_tsdp_get(tsd)); - /* Release all mutexes, now that fork() has completed. */ - stats_postfork_child(tsd_tsdn(tsd)); - for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { - arena_t *arena; - - if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { - arena_postfork_child(tsd_tsdn(tsd), arena); - } - } - prof_postfork_child(tsd_tsdn(tsd)); - if (have_background_thread) { - background_thread_postfork_child(tsd_tsdn(tsd)); - } - arenas_management_postfork_child(tsd_tsdn(tsd)); - tcache_postfork_child(tsd_tsdn(tsd)); - ctl_postfork_child(tsd_tsdn(tsd)); -} - -/******************************************************************************/ diff --git a/src/jemalloc_fork.c b/src/jemalloc_fork.c new file mode 100644 index 00000000..bd41383c --- /dev/null +++ b/src/jemalloc_fork.c @@ -0,0 +1,163 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/arenas_management.h" +#include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/jemalloc_fork.h" +#include "jemalloc/internal/jemalloc_init.h" + +/******************************************************************************/ +/* + * The following functions are used by threading libraries for protection of + * malloc during fork(). + */ + +#ifndef JEMALLOC_MUTEX_INIT_CB +void +jemalloc_prefork(void) +#else +JEMALLOC_EXPORT void +_malloc_prefork(void) +#endif +{ + tsd_t *tsd; + unsigned i, j, narenas; + arena_t *arena; + +#ifdef JEMALLOC_MUTEX_INIT_CB + if (!malloc_initialized()) { + return; + } +#endif + assert(malloc_initialized()); + + tsd = tsd_fetch(); + + narenas = narenas_total_get(); + + witness_prefork(tsd_witness_tsdp_get(tsd)); + /* Acquire all mutexes in a safe order. */ + ctl_prefork(tsd_tsdn(tsd)); + tcache_prefork(tsd_tsdn(tsd)); + arenas_management_prefork(tsd_tsdn(tsd)); + if (have_background_thread) { + background_thread_prefork0(tsd_tsdn(tsd)); + } + prof_prefork0(tsd_tsdn(tsd)); + if (have_background_thread) { + background_thread_prefork1(tsd_tsdn(tsd)); + } + /* Break arena prefork into stages to preserve lock order. */ + for (i = 0; i < 9; i++) { + for (j = 0; j < narenas; j++) { + if ((arena = arena_get(tsd_tsdn(tsd), j, false)) + != NULL) { + switch (i) { + case 0: + arena_prefork0(tsd_tsdn(tsd), arena); + break; + case 1: + arena_prefork1(tsd_tsdn(tsd), arena); + break; + case 2: + arena_prefork2(tsd_tsdn(tsd), arena); + break; + case 3: + arena_prefork3(tsd_tsdn(tsd), arena); + break; + case 4: + arena_prefork4(tsd_tsdn(tsd), arena); + break; + case 5: + arena_prefork5(tsd_tsdn(tsd), arena); + break; + case 6: + arena_prefork6(tsd_tsdn(tsd), arena); + break; + case 7: + arena_prefork7(tsd_tsdn(tsd), arena); + break; + case 8: + arena_prefork8(tsd_tsdn(tsd), arena); + break; + default: + not_reached(); + } + } + } + } + prof_prefork1(tsd_tsdn(tsd)); + stats_prefork(tsd_tsdn(tsd)); + tsd_prefork(tsd); +} + +#ifndef JEMALLOC_MUTEX_INIT_CB +void +jemalloc_postfork_parent(void) +#else +JEMALLOC_EXPORT void +_malloc_postfork(void) +#endif +{ + tsd_t *tsd; + unsigned i, narenas; + +#ifdef JEMALLOC_MUTEX_INIT_CB + if (!malloc_initialized()) { + return; + } +#endif + assert(malloc_initialized()); + + tsd = tsd_fetch(); + + tsd_postfork_parent(tsd); + + witness_postfork_parent(tsd_witness_tsdp_get(tsd)); + /* Release all mutexes, now that fork() has completed. */ + stats_postfork_parent(tsd_tsdn(tsd)); + for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { + arena_t *arena; + + if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { + arena_postfork_parent(tsd_tsdn(tsd), arena); + } + } + prof_postfork_parent(tsd_tsdn(tsd)); + if (have_background_thread) { + background_thread_postfork_parent(tsd_tsdn(tsd)); + } + arenas_management_postfork_parent(tsd_tsdn(tsd)); + tcache_postfork_parent(tsd_tsdn(tsd)); + ctl_postfork_parent(tsd_tsdn(tsd)); +} + +void +jemalloc_postfork_child(void) { + tsd_t *tsd; + unsigned i, narenas; + + assert(malloc_initialized()); + + tsd = tsd_fetch(); + + tsd_postfork_child(tsd); + + witness_postfork_child(tsd_witness_tsdp_get(tsd)); + /* Release all mutexes, now that fork() has completed. */ + stats_postfork_child(tsd_tsdn(tsd)); + for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { + arena_t *arena; + + if ((arena = arena_get(tsd_tsdn(tsd), i, false)) != NULL) { + arena_postfork_child(tsd_tsdn(tsd), arena); + } + } + prof_postfork_child(tsd_tsdn(tsd)); + if (have_background_thread) { + background_thread_postfork_child(tsd_tsdn(tsd)); + } + arenas_management_postfork_child(tsd_tsdn(tsd)); + tcache_postfork_child(tsd_tsdn(tsd)); + ctl_postfork_child(tsd_tsdn(tsd)); +} diff --git a/src/jemalloc_init.c b/src/jemalloc_init.c index 37e1350f..7ef5c77d 100644 --- a/src/jemalloc_init.c +++ b/src/jemalloc_init.c @@ -8,6 +8,7 @@ #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/hook.h" +#include "jemalloc/internal/jemalloc_fork.h" #include "jemalloc/internal/jemalloc_init.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" @@ -685,3 +686,24 @@ malloc_init_hard(void) { #undef UNLOCK_RETURN return false; } + +/* + * If an application creates a thread before doing any allocation in the main + * thread, then calls fork(2) in the main thread followed by memory allocation + * in the child process, a race can occur that results in deadlock within the + * child: the main thread may have forked while the created thread had + * partially initialized the allocator. Ordinarily jemalloc prevents + * fork/malloc races via the following functions it registers during + * initialization using pthread_atfork(), but of course that does no good if + * the allocator isn't fully initialized at fork time. The following library + * constructor is a partial solution to this problem. It may still be possible + * to trigger the deadlock described above, but doing so would involve forking + * via a library constructor that runs before jemalloc's runs. + */ +#ifndef JEMALLOC_JET +JEMALLOC_ATTR(constructor) +static void +jemalloc_constructor(void) { + malloc_init(); +} +#endif diff --git a/src/zone.c b/src/zone.c index e09de4b8..62d2eabb 100644 --- a/src/zone.c +++ b/src/zone.c @@ -2,6 +2,7 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/jemalloc_fork.h" #ifndef JEMALLOC_ZONE # error "This source file is for zones on Darwin (OS X)."