diff --git a/.github/workflows/linux-ci.yml b/.github/workflows/linux-ci.yml index 106f9600..99b5e50a 100644 --- a/.github/workflows/linux-ci.yml +++ b/.github/workflows/linux-ci.yml @@ -717,4 +717,23 @@ jobs: make check + test-linux-lto: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Install clang, lld and llvm + run: | + sudo apt-get update + sudo apt-get install -y clang lld llvm + + - name: Build and test (whole-program ThinLTO, je_ prefix) + run: | + autoconf + CC=clang AR=llvm-ar NM=llvm-nm RANLIB=llvm-ranlib \ + ./configure --with-jemalloc-prefix=je_ EXTRA_CFLAGS=-flto=thin + make -j3 EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld" + make -j3 tests EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld" + make check + diff --git a/Makefile.in b/Makefile.in index 75cc184a..f9c6c482 100644 --- a/Makefile.in +++ b/Makefile.in @@ -59,6 +59,7 @@ enable_autogen := @enable_autogen@ enable_doc := @enable_doc@ enable_shared := @enable_shared@ enable_static := @enable_static@ +have_ucontext := @have_ucontext@ enable_prof := @enable_prof@ enable_zone_allocator := @enable_zone_allocator@ enable_experimental_smallocx := @enable_experimental_smallocx@ @@ -348,6 +349,24 @@ ifeq (@enable_experimental_smallocx@, 1) TESTS_INTEGRATION += \ $(srcroot)test/integration/smallocx.c endif +# tcache_fiber_migration is a standalone LTO reproducer (issue #2890): the +# dedicated rule below links it against the static archive (--whole-archive) so +# the allocator fastpath inlines next to the swapcontext. It needs the static +# archive, GNU-ld --whole-archive semantics (the ELF gate), and working +# ucontext fibers -- have_ucontext is a configure link test because musl +# declares getcontext/makecontext/swapcontext but does not implement them, so +# the ELF gate alone would break `make check` on Alpine/musl. Built only when +# the build uses LTO (-flto in CFLAGS): without inlining across the swapcontext +# the bug cannot reproduce. +ifeq (elf, $(ABI)) +ifeq ($(enable_static), 1) +ifeq (1, $(have_ucontext)) +ifneq (,$(findstring -flto,$(CFLAGS))) +TESTS_INTEGRATION += $(srcroot)test/integration/tcache_fiber_migration.c +endif +endif +endif +endif ifeq (@enable_cxx@, 1) CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp @@ -581,6 +600,14 @@ $(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLI @mkdir -p $(@D) $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -pthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS) +# tcache_fiber_migration (issue #2890) must inline the allocator next to the +# swapcontext, so link jemalloc statically (--whole-archive), without the test +# harness/shared lib; whole-program LTO (from the build's flags) does the +# inlining. Explicit rule -- overrides the generic integration rule above. +$(objroot)test/integration/tcache_fiber_migration$(EXE): $(objroot)test/integration/tcache_fiber_migration.$(O) $(objroot)lib/$(LIBJEMALLOC).$(A) + @mkdir -p $(@D) + $(CC) $(LDTARGET) $(objroot)test/integration/tcache_fiber_migration.$(O) -Wl,--whole-archive $(objroot)lib/$(LIBJEMALLOC).$(A) -Wl,--no-whole-archive $(LDFLAGS) -pthread $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS) + $(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) $(CXX) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS) diff --git a/configure.ac b/configure.ac index b43fbc09..adb13429 100644 --- a/configure.ac +++ b/configure.ac @@ -1219,6 +1219,25 @@ if test "$enable_shared$enable_static" = "00" ; then AC_MSG_ERROR([Please enable one of shared or static builds]) fi +dnl Whether ucontext fibers (getcontext/makecontext/swapcontext) actually link. +dnl musl declares them in but exports no implementation, so a +dnl link test is required -- a header check would pass and then fail to link. +dnl Used to gate the tcache_fiber_migration test. +JE_COMPILABLE([ucontext], [ +#include +], [ + ucontext_t uc, ret; + getcontext(&uc); + makecontext(&uc, (void (*)(void))0, 0); + swapcontext(&ret, &uc); +], [je_cv_ucontext]) +if test "x${je_cv_ucontext}" = "xyes" ; then + have_ucontext="1" +else + have_ucontext="0" +fi +AC_SUBST([have_ucontext]) + dnl Perform no name mangling by default. AC_ARG_WITH([mangling], [AS_HELP_STRING([--with-mangling=], [Mangle symbols in ])], @@ -2861,6 +2880,8 @@ if test "x${je_cv_tls_model}" = "xyes" -a \ AC_DEFINE([JEMALLOC_TLS_MODEL], [__attribute__((tls_model("initial-exec")))], [ ]) + AC_DEFINE([JEMALLOC_TLS_MODEL_INITIAL_EXEC], [ ], + [Defined when the TSD thread-locals use the initial-exec (static) TLS model, i.e. live at a fixed offset from the thread pointer.]) else AC_DEFINE([JEMALLOC_TLS_MODEL], [ ], [ ]) fi diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index d64e08c3..d69e79bc 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -151,6 +151,13 @@ /* Non-empty if the tls_model attribute is supported. */ #undef JEMALLOC_TLS_MODEL +/* + * Defined when the TSD thread-locals use the initial-exec (static) TLS model, + * i.e. live at a fixed offset from the thread pointer. Gates the fast path of + * JEMALLOC_TLS_ADDR (see tsd_internals.h). + */ +#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC + /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables * inline functions. diff --git a/include/jemalloc/internal/tsd_internals.h b/include/jemalloc/internal/tsd_internals.h index b318c464..4284787d 100644 --- a/include/jemalloc/internal/tsd_internals.h +++ b/include/jemalloc/internal/tsd_internals.h @@ -25,6 +25,97 @@ typedef struct arena_s arena_t; typedef struct prof_tdata_s prof_tdata_t; +/* + * JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler + * cannot cache it across a user-space context switch. A raw `&tlsvar` is + * `thread_pointer + const_offset`, loop-invariant; inlined into malloc/free + * under LTO it can be hoisted across a swapcontext and reused on the OS thread a + * fiber migrated to -- a stale tsd/tcache. Route all tsd access through this + * macro; never take `&tsd_tls` raw. See + * https://github.com/jemalloc/jemalloc/issues/2890 + * + * Static-TLS fast path (gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC): re-read the + * thread pointer with a volatile asm and add a runtime-captured constant offset. + * The offset MUST be captured by the noinline helper, not computed inline as + * `&tlsvar - thread_pointer` -- inline, the two terms hoist independently and + * cancel the volatile read back to the stale address. The offset is read and + * written with relaxed atomics -- threads racing their first allocation init it + * concurrently with the same thread-independent value. Other configs use a + * noinline `memory`-barrier accessor. MSVC has no inline asm, so it keeps the + * plain address; an MSVC build whose fibers run under whole-program opt (/GL) + * must instead compile with /GT (fiber-safe TLS). + * + * DECLARE is emitted in every TU; DEFINE (the out-of-line bodies) only under + * JEMALLOC_TSD_C_, i.e. once in src/tsd.c. Both must be invoked directly, not + * forwarded through another macro, or `##tlsvar` pastes the macro-expanded + * `je_tsd_tls` instead of the literal name. + */ +#if defined(__GNUC__) && !defined(_WIN32) && \ + defined(JEMALLOC_TLS_MODEL_INITIAL_EXEC) && \ + (defined(__aarch64__) || defined(__arm__) || defined(__x86_64__) || \ + defined(__i386__)) +JEMALLOC_ALWAYS_INLINE char * +jemalloc_thread_pointer(void) { + char *thread_pointer; +# if defined(__aarch64__) && defined(__APPLE__) + __asm__ __volatile__("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r"(thread_pointer)); +# elif defined(__aarch64__) + __asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(thread_pointer)); +# elif defined(__arm__) + __asm__ __volatile__("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r"(thread_pointer)); +# elif defined(__x86_64__) && defined(__APPLE__) + __asm__ __volatile__("movq %%gs:0, %0" : "=r"(thread_pointer)); +# elif defined(__x86_64__) + __asm__ __volatile__("movq %%fs:0, %0" : "=r"(thread_pointer)); +# else /* __i386__ */ + __asm__ __volatile__("movl %%gs:0, %0" : "=r"(thread_pointer)); +# endif + return thread_pointer; +} +/* 1 is unreachable: tlsvar and the thread pointer are at least 4-aligned. */ +# define JEMALLOC_TLS_OFFSET_UNINITIALIZED 1 +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ + extern intptr_t jemalloc_tls_offset_##tlsvar; \ + intptr_t jemalloc_tls_offset_init_##tlsvar(void); \ + JEMALLOC_ALWAYS_INLINE __typeof__(&(tlsvar)) \ + jemalloc_tls_addr_##tlsvar(void) { \ + intptr_t tls_offset = __atomic_load_n(&jemalloc_tls_offset_##tlsvar, \ + __ATOMIC_RELAXED); \ + if (unlikely(tls_offset == JEMALLOC_TLS_OFFSET_UNINITIALIZED)) { \ + tls_offset = jemalloc_tls_offset_init_##tlsvar(); \ + } \ + return (__typeof__(&(tlsvar)))(jemalloc_thread_pointer() + \ + tls_offset); \ + } +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \ + intptr_t jemalloc_tls_offset_##tlsvar = \ + JEMALLOC_TLS_OFFSET_UNINITIALIZED; \ + JEMALLOC_NOINLINE intptr_t \ + jemalloc_tls_offset_init_##tlsvar(void) { \ + intptr_t tls_offset = (intptr_t)((char *)&(tlsvar) - \ + jemalloc_thread_pointer()); \ + __atomic_store_n(&jemalloc_tls_offset_##tlsvar, tls_offset, \ + __ATOMIC_RELAXED); \ + return tls_offset; \ + } +# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar()) +#elif defined(__GNUC__) +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) \ + __typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void); +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) \ + JEMALLOC_NOINLINE __typeof__(&(tlsvar)) \ + jemalloc_tls_addr_##tlsvar(void) { \ + __typeof__(&(tlsvar)) tls_addr = &(tlsvar); \ + __asm__ __volatile__("" : "+r"(tls_addr) : : "memory"); \ + return tls_addr; \ + } +# define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar()) +#else +# define JEMALLOC_TLS_ADDR_DECLARE(tlsvar) +# define JEMALLOC_TLS_ADDR_DEFINE(tlsvar) +# define JEMALLOC_TLS_ADDR(tlsvar) (&(tlsvar)) +#endif + /* * Thread-Specific-Data layout * diff --git a/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index e1d46de8..b3ae4835 100644 --- a/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -13,6 +13,13 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_tls) +JEMALLOC_TLS_ADDR_DECLARE(tsd_initialized) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_tls) +JEMALLOC_TLS_ADDR_DEFINE(tsd_initialized) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_cleanup_wrapper(void) { @@ -58,13 +65,15 @@ tsd_teardown_done(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &tsd_tls; + return JEMALLOC_TLS_ADDR(tsd_tls); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls); + assert(tsd_booted); - if (likely(&tsd_tls != val)) { - tsd_tls = (*val); + if (likely(tsd != val)) { + *tsd = (*val); } - tsd_initialized = true; + *JEMALLOC_TLS_ADDR(tsd_initialized) = true; } diff --git a/include/jemalloc/internal/tsd_tls.h b/include/jemalloc/internal/tsd_tls.h index 15b718d2..3a91d312 100644 --- a/include/jemalloc/internal/tsd_tls.h +++ b/include/jemalloc/internal/tsd_tls.h @@ -13,6 +13,11 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls; extern pthread_key_t tsd_tsd; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_tls) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_tls) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_boot0(void) { @@ -51,16 +56,18 @@ tsd_teardown_done(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &tsd_tls; + return JEMALLOC_TLS_ADDR(tsd_tls); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls); + assert(tsd_booted); - if (likely(&tsd_tls != val)) { - tsd_tls = (*val); + if (likely(tsd != val)) { + *tsd = (*val); } - if (pthread_setspecific(tsd_tsd, (void *)(&tsd_tls)) != 0) { + if (pthread_setspecific(tsd_tsd, (void *)tsd) != 0) { malloc_write(": Error setting tsd.\n"); if (opt_abort) { abort(); diff --git a/include/jemalloc/internal/tsd_win.h b/include/jemalloc/internal/tsd_win.h index 837f0f89..4a827879 100644 --- a/include/jemalloc/internal/tsd_win.h +++ b/include/jemalloc/internal/tsd_win.h @@ -183,6 +183,11 @@ tsd_set(tsd_t *val) { extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls; extern bool tsd_booted; +JEMALLOC_TLS_ADDR_DECLARE(tsd_wrapper_tls) +#ifdef JEMALLOC_TSD_C_ +JEMALLOC_TLS_ADDR_DEFINE(tsd_wrapper_tls) +#endif + /* Initialization/cleanup. */ JEMALLOC_ALWAYS_INLINE bool tsd_cleanup_wrapper(void) { @@ -233,16 +238,18 @@ tsd_teardown_done(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - return &(tsd_wrapper_tls.val); + return &(JEMALLOC_TLS_ADDR(tsd_wrapper_tls)->val); } JEMALLOC_ALWAYS_INLINE void tsd_set(tsd_t *val) { + tsd_wrapper_t *wrapper = JEMALLOC_TLS_ADDR(tsd_wrapper_tls); + assert(tsd_booted); - if (likely(&(tsd_wrapper_tls.val) != val)) { - tsd_wrapper_tls.val = (*val); + if (likely(&wrapper->val != val)) { + wrapper->val = (*val); } - tsd_wrapper_tls.initialized = true; + wrapper->initialized = true; } #endif // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER) diff --git a/scripts/gen_gh_actions.py b/scripts/gen_gh_actions.py index e35196d5..fc1f2e6d 100755 --- a/scripts/gen_gh_actions.py +++ b/scripts/gen_gh_actions.py @@ -706,6 +706,34 @@ def generate_freebsd_job(arch): return job +def generate_linux_lto_job(): + """Dedicated lane: whole-program ThinLTO + the je_ public prefix, statically + linked. This is the configuration under which the tcache_fiber_migration + reproducer (issue #2890) actually exercises the bug -- the allocator + fastpath must be inlined next to the swapcontext, which only happens with + static linking under LTO. llvm-ar/nm/ranlib are needed to archive the LTO + bitcode; -fuse-ld=lld to link it.""" + return """ test-linux-lto: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Install clang, lld and llvm + run: | + sudo apt-get update + sudo apt-get install -y clang lld llvm + + - name: Build and test (whole-program ThinLTO, je_ prefix) + run: | + autoconf + CC=clang AR=llvm-ar NM=llvm-nm RANLIB=llvm-ranlib \\ + ./configure --with-jemalloc-prefix=je_ EXTRA_CFLAGS=-flto=thin + make -j3 EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld" + make -j3 tests EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld" + make check +""" + + def main(): import sys @@ -716,6 +744,7 @@ def main(): jobs = '\n'.join(( generate_linux_job(AMD64), generate_linux_job(ARM64), + generate_linux_lto_job(), )) print(GITHUB_ACTIONS_TEMPLATE.format(name='Linux CI', jobs=jobs)) @@ -739,6 +768,7 @@ def main(): linux_jobs = '\n'.join(( generate_linux_job(AMD64), generate_linux_job(ARM64), + generate_linux_lto_job(), )) macos_jobs = '\n'.join(( generate_macos_job(AMD64), # Intel diff --git a/src/tsd.c b/src/tsd.c index 849f2297..43547c98 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -1,3 +1,6 @@ +/* Emit the single out-of-line JEMALLOC_TLS_ADDR offset globals/helpers here. */ +#define JEMALLOC_TSD_C_ + #include "jemalloc/internal/jemalloc_preamble.h" #include "jemalloc/internal/arenas_management.h" diff --git a/test/integration/tcache_fiber_migration.c b/test/integration/tcache_fiber_migration.c new file mode 100644 index 00000000..27698d37 --- /dev/null +++ b/test/integration/tcache_fiber_migration.c @@ -0,0 +1,135 @@ +/* + * Regression test for issue #2890: under whole-program LTO the inlined allocator + * fastpath caches the TLS-derived tcache base in a register across a + * swapcontext, so a fiber resumed on another OS thread uses the previous + * thread's tcache -> heap corruption. See JEMALLOC_TLS_ADDR in tsd_internals.h. + * + * Standalone (no test harness, so it can be static-linked), calling jemalloc via + * JEMALLOC_MANGLE -- it rewrites malloc/free to jemalloc's configured-prefix + * symbols, so the calls bind to and inline the static allocator, not libc's + * wrappers. Reproduces only with the allocator inlined next to the swapcontext + * (a static --whole-archive link plus whole-program LTO), so the Makefile builds + * it only when -flto is in the build flags. + */ +#include +#include +#include + +#define JEMALLOC_MANGLE +#include + +enum { + num_fibers = 128, + num_workers = 8, + ops_per_fiber = 20 * 1000, + fiber_stack_size = 1 << 16, + ready_queue_capacity = 512 /* power of two, > num_fibers */ +}; + +static ucontext_t fiber_context[num_fibers]; +/* Scheduler context to switch back to when a fiber yields; the resuming worker + * writes its own context here, so this changes when the fiber migrates. */ +static ucontext_t *return_context[num_fibers]; +static unsigned fiber_remaining_ops[num_fibers]; +static bool fiber_done[num_fibers]; + +static unsigned ready_queue[ready_queue_capacity]; +static unsigned ready_queue_head; +static unsigned ready_queue_tail; +static unsigned live_fibers; +static pthread_mutex_t queue_mtx = PTHREAD_MUTEX_INITIALIZER; + +static void +push_ready_fiber(unsigned id) { + pthread_mutex_lock(&queue_mtx); + ready_queue[ready_queue_head++ & (ready_queue_capacity - 1)] = id; + pthread_mutex_unlock(&queue_mtx); +} + +/* Returns a ready fiber id, -1 if none ready now, or -2 if all have finished. */ +static int +pop_ready_fiber(void) { + int id; + pthread_mutex_lock(&queue_mtx); + if (live_fibers == 0) { + id = -2; + } else if (ready_queue_tail != ready_queue_head) { + id = (int)ready_queue[ready_queue_tail++ & (ready_queue_capacity - 1)]; + } else { + id = -1; + } + pthread_mutex_unlock(&queue_mtx); + return id; +} + +static void +fiber_run(int id) { + void *p = malloc(64); + while (fiber_remaining_ops[id] > 0) { + fiber_remaining_ops[id]--; + free(p); /* push to the CURRENT thread's tcache */ + /* Yield; may be resumed on a different worker thread. */ + swapcontext(&fiber_context[id], return_context[id]); + p = malloc(64); /* pop; must come from the NEW thread's tcache */ + *(char *)p = (char)id; + } + free(p); + pthread_mutex_lock(&queue_mtx); + fiber_done[id] = true; + live_fibers--; + pthread_mutex_unlock(&queue_mtx); + swapcontext(&fiber_context[id], return_context[id]); +} + +static void * +worker_thread(void *arg) { + ucontext_t scheduler_context; + (void)arg; + for (;;) { + int id = pop_ready_fiber(); + if (id == -2) { + break; + } + if (id < 0) { + continue; + } + return_context[id] = &scheduler_context; + swapcontext(&scheduler_context, &fiber_context[id]); + if (!fiber_done[id]) { + push_ready_fiber((unsigned)id); + } + } + return NULL; +} + +int +main(void) { + void *stacks[num_fibers]; + pthread_t threads[num_workers]; + unsigned i; + + live_fibers = num_fibers; + for (i = 0; i < num_fibers; i++) { + stacks[i] = malloc(fiber_stack_size); + fiber_remaining_ops[i] = ops_per_fiber; + getcontext(&fiber_context[i]); + fiber_context[i].uc_stack.ss_sp = stacks[i]; + fiber_context[i].uc_stack.ss_size = fiber_stack_size; + fiber_context[i].uc_link = NULL; + makecontext(&fiber_context[i], (void (*)(void))fiber_run, 1, (int)i); + push_ready_fiber(i); + } + + for (i = 0; i < num_workers; i++) { + pthread_create(&threads[i], NULL, worker_thread, NULL); + } + for (i = 0; i < num_workers; i++) { + pthread_join(threads[i], NULL); + } + for (i = 0; i < num_fibers; i++) { + free(stacks[i]); + } + + /* Completing without a tcache-corruption crash is success. */ + return live_fibers == 0 ? 0 : 1; +}