Merge 7aa77c8e2a into b44e84086f

.github/workflows/linux-ci.yml

@@ -717,4 +717,23 @@ jobs:
         make check
 
 
+  test-linux-lto:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install clang, lld and llvm
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang lld llvm
+
+    - name: Build and test (whole-program ThinLTO, je_ prefix)
+      run: |
+        autoconf
+        CC=clang AR=llvm-ar NM=llvm-nm RANLIB=llvm-ranlib \
+          ./configure --with-jemalloc-prefix=je_ EXTRA_CFLAGS=-flto=thin
+        make -j3 EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld"
+        make -j3 tests EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld"
+        make check
+
 

Makefile.in

@@ -59,6 +59,7 @@ enable_autogen := @enable_autogen@
 enable_doc := @enable_doc@
 enable_shared := @enable_shared@
 enable_static := @enable_static@
+have_ucontext := @have_ucontext@
 enable_prof := @enable_prof@
 enable_zone_allocator := @enable_zone_allocator@
 enable_experimental_smallocx := @enable_experimental_smallocx@
@@ -348,6 +349,24 @@ ifeq (@enable_experimental_smallocx@, 1)
 TESTS_INTEGRATION += \
   $(srcroot)test/integration/smallocx.c
 endif
+# tcache_fiber_migration is a standalone LTO reproducer (issue #2890): the
+# dedicated rule below links it against the static archive (--whole-archive) so
+# the allocator fastpath inlines next to the swapcontext.  It needs the static
+# archive, GNU-ld --whole-archive semantics (the ELF gate), and working
+# ucontext fibers -- have_ucontext is a configure link test because musl
+# declares getcontext/makecontext/swapcontext but does not implement them, so
+# the ELF gate alone would break `make check` on Alpine/musl.  Built only when
+# the build uses LTO (-flto in CFLAGS): without inlining across the swapcontext
+# the bug cannot reproduce.
+ifeq (elf, $(ABI))
+ifeq ($(enable_static), 1)
+ifeq (1, $(have_ucontext))
+ifneq (,$(findstring -flto,$(CFLAGS)))
+TESTS_INTEGRATION += $(srcroot)test/integration/tcache_fiber_migration.c
+endif
+endif
+endif
+endif
 ifeq (@enable_cxx@, 1)
 CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp
 TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp
@@ -581,6 +600,14 @@ $(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLI
 	@mkdir -p $(@D)
 	$(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -pthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS)
 
+# tcache_fiber_migration (issue #2890) must inline the allocator next to the
+# swapcontext, so link jemalloc statically (--whole-archive), without the test
+# harness/shared lib; whole-program LTO (from the build's flags) does the
+# inlining.  Explicit rule -- overrides the generic integration rule above.
+$(objroot)test/integration/tcache_fiber_migration$(EXE): $(objroot)test/integration/tcache_fiber_migration.$(O) $(objroot)lib/$(LIBJEMALLOC).$(A)
+	@mkdir -p $(@D)
+	$(CC) $(LDTARGET) $(objroot)test/integration/tcache_fiber_migration.$(O) -Wl,--whole-archive $(objroot)lib/$(LIBJEMALLOC).$(A) -Wl,--no-whole-archive $(LDFLAGS) -pthread $(filter-out -lm,$(LIBS)) $(LM) $(EXTRA_LDFLAGS)
+
 $(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB)
 	@mkdir -p $(@D)
 	$(CXX) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(LDFLAGS) $(filter-out -lm,$(LIBS)) -lm $(EXTRA_LDFLAGS)

configure.ac

@@ -1219,6 +1219,25 @@ if test "$enable_shared$enable_static" = "00" ; then
   AC_MSG_ERROR([Please enable one of shared or static builds])
 fi
 
+dnl Whether ucontext fibers (getcontext/makecontext/swapcontext) actually link.
+dnl musl declares them in <ucontext.h> but exports no implementation, so a
+dnl link test is required -- a header check would pass and then fail to link.
+dnl Used to gate the tcache_fiber_migration test.
+JE_COMPILABLE([ucontext], [
+#include <ucontext.h>
+], [
+	ucontext_t uc, ret;
+	getcontext(&uc);
+	makecontext(&uc, (void (*)(void))0, 0);
+	swapcontext(&ret, &uc);
+], [je_cv_ucontext])
+if test "x${je_cv_ucontext}" = "xyes" ; then
+  have_ucontext="1"
+else
+  have_ucontext="0"
+fi
+AC_SUBST([have_ucontext])
+
 dnl Perform no name mangling by default.
 AC_ARG_WITH([mangling],
   [AS_HELP_STRING([--with-mangling=<map>], [Mangle symbols in <map>])],
@@ -2861,6 +2880,8 @@ if test "x${je_cv_tls_model}" = "xyes" -a \
   AC_DEFINE([JEMALLOC_TLS_MODEL],
             [__attribute__((tls_model("initial-exec")))],
             [ ])
+  AC_DEFINE([JEMALLOC_TLS_MODEL_INITIAL_EXEC], [ ],
+            [Defined when the TSD thread-locals use the initial-exec (static) TLS model, i.e. live at a fixed offset from the thread pointer.])
 else
   AC_DEFINE([JEMALLOC_TLS_MODEL], [ ], [ ])
 fi

include/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -151,6 +151,13 @@
 /* Non-empty if the tls_model attribute is supported. */
 #undef JEMALLOC_TLS_MODEL
 
+/*
+ * Defined when the TSD thread-locals use the initial-exec (static) TLS model,
+ * i.e. live at a fixed offset from the thread pointer.  Gates the fast path of
+ * JEMALLOC_TLS_ADDR (see tsd_internals.h).
+ */
+#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC
+
 /*
  * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables
  * inline functions.

include/jemalloc/internal/tsd_internals.h

@@ -25,6 +25,97 @@
 typedef struct arena_s      arena_t;
 typedef struct prof_tdata_s prof_tdata_t;
 
+/*
+ * JEMALLOC_TLS_ADDR(tlsvar): take a thread-local's address so the compiler
+ * cannot cache it across a user-space context switch.  A raw `&tlsvar` is
+ * `thread_pointer + const_offset`, loop-invariant; inlined into malloc/free
+ * under LTO it can be hoisted across a swapcontext and reused on the OS thread a
+ * fiber migrated to -- a stale tsd/tcache.  Route all tsd access through this
+ * macro; never take `&tsd_tls` raw.  See
+ * https://github.com/jemalloc/jemalloc/issues/2890
+ *
+ * Static-TLS fast path (gated on JEMALLOC_TLS_MODEL_INITIAL_EXEC): re-read the
+ * thread pointer with a volatile asm and add a runtime-captured constant offset.
+ * The offset MUST be captured by the noinline helper, not computed inline as
+ * `&tlsvar - thread_pointer` -- inline, the two terms hoist independently and
+ * cancel the volatile read back to the stale address.  The offset is read and
+ * written with relaxed atomics -- threads racing their first allocation init it
+ * concurrently with the same thread-independent value.  Other configs use a
+ * noinline `memory`-barrier accessor.  MSVC has no inline asm, so it keeps the
+ * plain address; an MSVC build whose fibers run under whole-program opt (/GL)
+ * must instead compile with /GT (fiber-safe TLS).
+ *
+ * DECLARE is emitted in every TU; DEFINE (the out-of-line bodies) only under
+ * JEMALLOC_TSD_C_, i.e. once in src/tsd.c.  Both must be invoked directly, not
+ * forwarded through another macro, or `##tlsvar` pastes the macro-expanded
+ * `je_tsd_tls` instead of the literal name.
+ */
+#if defined(__GNUC__) && !defined(_WIN32) &&                                   \
+    defined(JEMALLOC_TLS_MODEL_INITIAL_EXEC) &&                                \
+    (defined(__aarch64__) || defined(__arm__) || defined(__x86_64__) ||        \
+    defined(__i386__))
+JEMALLOC_ALWAYS_INLINE char *
+jemalloc_thread_pointer(void) {
+	char *thread_pointer;
+#  if defined(__aarch64__) && defined(__APPLE__)
+	__asm__ __volatile__("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r"(thread_pointer));
+#  elif defined(__aarch64__)
+	__asm__ __volatile__("mrs %0, tpidr_el0" : "=r"(thread_pointer));
+#  elif defined(__arm__)
+	__asm__ __volatile__("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r"(thread_pointer));
+#  elif defined(__x86_64__) && defined(__APPLE__)
+	__asm__ __volatile__("movq %%gs:0, %0" : "=r"(thread_pointer));
+#  elif defined(__x86_64__)
+	__asm__ __volatile__("movq %%fs:0, %0" : "=r"(thread_pointer));
+#  else /* __i386__ */
+	__asm__ __volatile__("movl %%gs:0, %0" : "=r"(thread_pointer));
+#  endif
+	return thread_pointer;
+}
+/* 1 is unreachable: tlsvar and the thread pointer are at least 4-aligned. */
+#  define JEMALLOC_TLS_OFFSET_UNINITIALIZED 1
+#  define JEMALLOC_TLS_ADDR_DECLARE(tlsvar)                                    \
+	extern intptr_t jemalloc_tls_offset_##tlsvar;                                 \
+	intptr_t jemalloc_tls_offset_init_##tlsvar(void);                             \
+	JEMALLOC_ALWAYS_INLINE __typeof__(&(tlsvar))                                  \
+	jemalloc_tls_addr_##tlsvar(void) {                                            \
+		intptr_t tls_offset = __atomic_load_n(&jemalloc_tls_offset_##tlsvar,         \
+		    __ATOMIC_RELAXED);                                                       \
+		if (unlikely(tls_offset == JEMALLOC_TLS_OFFSET_UNINITIALIZED)) {             \
+			tls_offset = jemalloc_tls_offset_init_##tlsvar();                           \
+		}                                                                            \
+		return (__typeof__(&(tlsvar)))(jemalloc_thread_pointer() +                   \
+		    tls_offset);                                                             \
+	}
+#  define JEMALLOC_TLS_ADDR_DEFINE(tlsvar)                                     \
+	intptr_t jemalloc_tls_offset_##tlsvar =                                       \
+	    JEMALLOC_TLS_OFFSET_UNINITIALIZED;                                        \
+	JEMALLOC_NOINLINE intptr_t                                                    \
+	jemalloc_tls_offset_init_##tlsvar(void) {                                     \
+		intptr_t tls_offset = (intptr_t)((char *)&(tlsvar) -                         \
+		    jemalloc_thread_pointer());                                              \
+		__atomic_store_n(&jemalloc_tls_offset_##tlsvar, tls_offset,                  \
+		    __ATOMIC_RELAXED);                                                       \
+		return tls_offset;                                                           \
+	}
+#  define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar())
+#elif defined(__GNUC__)
+#  define JEMALLOC_TLS_ADDR_DECLARE(tlsvar)                                    \
+	__typeof__(&(tlsvar)) jemalloc_tls_addr_##tlsvar(void);
+#  define JEMALLOC_TLS_ADDR_DEFINE(tlsvar)                                     \
+	JEMALLOC_NOINLINE __typeof__(&(tlsvar))                                       \
+	jemalloc_tls_addr_##tlsvar(void) {                                            \
+		__typeof__(&(tlsvar)) tls_addr = &(tlsvar);                                  \
+		__asm__ __volatile__("" : "+r"(tls_addr) : : "memory");                      \
+		return tls_addr;                                                             \
+	}
+#  define JEMALLOC_TLS_ADDR(tlsvar) (jemalloc_tls_addr_##tlsvar())
+#else
+#  define JEMALLOC_TLS_ADDR_DECLARE(tlsvar)
+#  define JEMALLOC_TLS_ADDR_DEFINE(tlsvar)
+#  define JEMALLOC_TLS_ADDR(tlsvar) (&(tlsvar))
+#endif
+
 /*
  * Thread-Specific-Data layout
  *

include/jemalloc/internal/tsd_malloc_thread_cleanup.h

@@ -13,6 +13,13 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls;
 extern JEMALLOC_TSD_TYPE_ATTR(bool) tsd_initialized;
 extern bool tsd_booted;
 
+JEMALLOC_TLS_ADDR_DECLARE(tsd_tls)
+JEMALLOC_TLS_ADDR_DECLARE(tsd_initialized)
+#ifdef JEMALLOC_TSD_C_
+JEMALLOC_TLS_ADDR_DEFINE(tsd_tls)
+JEMALLOC_TLS_ADDR_DEFINE(tsd_initialized)
+#endif
+
 /* Initialization/cleanup. */
 JEMALLOC_ALWAYS_INLINE bool
 tsd_cleanup_wrapper(void) {
@@ -58,13 +65,15 @@ tsd_teardown_done(void) {
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
-	return &tsd_tls;
+	return JEMALLOC_TLS_ADDR(tsd_tls);
 }
 JEMALLOC_ALWAYS_INLINE void
 tsd_set(tsd_t *val) {
+	tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls);
+
 	assert(tsd_booted);
-	if (likely(&tsd_tls != val)) {
-		tsd_tls = (*val);
+	if (likely(tsd != val)) {
+		*tsd = (*val);
 	}
-	tsd_initialized = true;
+	*JEMALLOC_TLS_ADDR(tsd_initialized) = true;
 }

include/jemalloc/internal/tsd_tls.h

@@ -13,6 +13,11 @@ extern JEMALLOC_TSD_TYPE_ATTR(tsd_t) tsd_tls;
 extern pthread_key_t tsd_tsd;
 extern bool          tsd_booted;
 
+JEMALLOC_TLS_ADDR_DECLARE(tsd_tls)
+#ifdef JEMALLOC_TSD_C_
+JEMALLOC_TLS_ADDR_DEFINE(tsd_tls)
+#endif
+
 /* Initialization/cleanup. */
 JEMALLOC_ALWAYS_INLINE bool
 tsd_boot0(void) {
@@ -51,16 +56,18 @@ tsd_teardown_done(void) {
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
-	return &tsd_tls;
+	return JEMALLOC_TLS_ADDR(tsd_tls);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 tsd_set(tsd_t *val) {
+	tsd_t *tsd = JEMALLOC_TLS_ADDR(tsd_tls);
+
 	assert(tsd_booted);
-	if (likely(&tsd_tls != val)) {
-		tsd_tls = (*val);
+	if (likely(tsd != val)) {
+		*tsd = (*val);
 	}
-	if (pthread_setspecific(tsd_tsd, (void *)(&tsd_tls)) != 0) {
+	if (pthread_setspecific(tsd_tsd, (void *)tsd) != 0) {
 		malloc_write("<jemalloc>: Error setting tsd.\n");
 		if (opt_abort) {
 			abort();

include/jemalloc/internal/tsd_win.h

@@ -183,6 +183,11 @@ tsd_set(tsd_t *val) {
 extern JEMALLOC_TSD_TYPE_ATTR(tsd_wrapper_t) tsd_wrapper_tls;
 extern bool tsd_booted;
 
+JEMALLOC_TLS_ADDR_DECLARE(tsd_wrapper_tls)
+#ifdef JEMALLOC_TSD_C_
+JEMALLOC_TLS_ADDR_DEFINE(tsd_wrapper_tls)
+#endif
+
 /* Initialization/cleanup. */
 JEMALLOC_ALWAYS_INLINE bool
 tsd_cleanup_wrapper(void) {
@@ -233,16 +238,18 @@ tsd_teardown_done(void) {
 /* Get/set. */
 JEMALLOC_ALWAYS_INLINE tsd_t *
 tsd_get(bool init) {
-	return &(tsd_wrapper_tls.val);
+	return &(JEMALLOC_TLS_ADDR(tsd_wrapper_tls)->val);
 }
 
 JEMALLOC_ALWAYS_INLINE void
 tsd_set(tsd_t *val) {
+	tsd_wrapper_t *wrapper = JEMALLOC_TLS_ADDR(tsd_wrapper_tls);
+
 	assert(tsd_booted);
-	if (likely(&(tsd_wrapper_tls.val) != val)) {
-		tsd_wrapper_tls.val = (*val);
+	if (likely(&wrapper->val != val)) {
+		wrapper->val = (*val);
 	}
-	tsd_wrapper_tls.initialized = true;
+	wrapper->initialized = true;
 }
 
 #endif // defined(JEMALLOC_LEGACY_WINDOWS_SUPPORT) || !defined(_MSC_VER)

scripts/gen_gh_actions.py

@@ -706,6 +706,34 @@ def generate_freebsd_job(arch):
     return job
 
 
+def generate_linux_lto_job():
+    """Dedicated lane: whole-program ThinLTO + the je_ public prefix, statically
+    linked.  This is the configuration under which the tcache_fiber_migration
+    reproducer (issue #2890) actually exercises the bug -- the allocator
+    fastpath must be inlined next to the swapcontext, which only happens with
+    static linking under LTO.  llvm-ar/nm/ranlib are needed to archive the LTO
+    bitcode; -fuse-ld=lld to link it."""
+    return """  test-linux-lto:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install clang, lld and llvm
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y clang lld llvm
+
+    - name: Build and test (whole-program ThinLTO, je_ prefix)
+      run: |
+        autoconf
+        CC=clang AR=llvm-ar NM=llvm-nm RANLIB=llvm-ranlib \\
+          ./configure --with-jemalloc-prefix=je_ EXTRA_CFLAGS=-flto=thin
+        make -j3 EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld"
+        make -j3 tests EXTRA_LDFLAGS="-flto=thin -fuse-ld=lld"
+        make check
+"""
+
+
 def main():
     import sys
 
@@ -716,6 +744,7 @@ def main():
         jobs = '\n'.join((
             generate_linux_job(AMD64),
             generate_linux_job(ARM64),
+            generate_linux_lto_job(),
         ))
         print(GITHUB_ACTIONS_TEMPLATE.format(name='Linux CI', jobs=jobs))
 
@@ -739,6 +768,7 @@ def main():
         linux_jobs = '\n'.join((
             generate_linux_job(AMD64),
             generate_linux_job(ARM64),
+            generate_linux_lto_job(),
         ))
         macos_jobs = '\n'.join((
             generate_macos_job(AMD64),   # Intel

src/tsd.c

@@ -1,3 +1,6 @@
+/* Emit the single out-of-line JEMALLOC_TLS_ADDR offset globals/helpers here. */
+#define JEMALLOC_TSD_C_
+
 #include "jemalloc/internal/jemalloc_preamble.h"
 
 #include "jemalloc/internal/arenas_management.h"

test/integration/tcache_fiber_migration.c

@@ -0,0 +1,135 @@
+/*
+ * Regression test for issue #2890: under whole-program LTO the inlined allocator
+ * fastpath caches the TLS-derived tcache base in a register across a
+ * swapcontext, so a fiber resumed on another OS thread uses the previous
+ * thread's tcache -> heap corruption.  See JEMALLOC_TLS_ADDR in tsd_internals.h.
+ *
+ * Standalone (no test harness, so it can be static-linked), calling jemalloc via
+ * JEMALLOC_MANGLE -- it rewrites malloc/free to jemalloc's configured-prefix
+ * symbols, so the calls bind to and inline the static allocator, not libc's
+ * wrappers.  Reproduces only with the allocator inlined next to the swapcontext
+ * (a static --whole-archive link plus whole-program LTO), so the Makefile builds
+ * it only when -flto is in the build flags.
+ */
+#include <pthread.h>
+#include <stdbool.h>
+#include <ucontext.h>
+
+#define JEMALLOC_MANGLE
+#include <jemalloc/jemalloc.h>
+
+enum {
+	num_fibers = 128,
+	num_workers = 8,
+	ops_per_fiber = 20 * 1000,
+	fiber_stack_size = 1 << 16,
+	ready_queue_capacity = 512 /* power of two, > num_fibers */
+};
+
+static ucontext_t fiber_context[num_fibers];
+/* Scheduler context to switch back to when a fiber yields; the resuming worker
+ * writes its own context here, so this changes when the fiber migrates. */
+static ucontext_t *return_context[num_fibers];
+static unsigned fiber_remaining_ops[num_fibers];
+static bool fiber_done[num_fibers];
+
+static unsigned ready_queue[ready_queue_capacity];
+static unsigned ready_queue_head;
+static unsigned ready_queue_tail;
+static unsigned live_fibers;
+static pthread_mutex_t queue_mtx = PTHREAD_MUTEX_INITIALIZER;
+
+static void
+push_ready_fiber(unsigned id) {
+	pthread_mutex_lock(&queue_mtx);
+	ready_queue[ready_queue_head++ & (ready_queue_capacity - 1)] = id;
+	pthread_mutex_unlock(&queue_mtx);
+}
+
+/* Returns a ready fiber id, -1 if none ready now, or -2 if all have finished. */
+static int
+pop_ready_fiber(void) {
+	int id;
+	pthread_mutex_lock(&queue_mtx);
+	if (live_fibers == 0) {
+		id = -2;
+	} else if (ready_queue_tail != ready_queue_head) {
+		id = (int)ready_queue[ready_queue_tail++ & (ready_queue_capacity - 1)];
+	} else {
+		id = -1;
+	}
+	pthread_mutex_unlock(&queue_mtx);
+	return id;
+}
+
+static void
+fiber_run(int id) {
+	void *p = malloc(64);
+	while (fiber_remaining_ops[id] > 0) {
+		fiber_remaining_ops[id]--;
+		free(p); /* push to the CURRENT thread's tcache */
+		/* Yield; may be resumed on a different worker thread. */
+		swapcontext(&fiber_context[id], return_context[id]);
+		p = malloc(64); /* pop; must come from the NEW thread's tcache */
+		*(char *)p = (char)id;
+	}
+	free(p);
+	pthread_mutex_lock(&queue_mtx);
+	fiber_done[id] = true;
+	live_fibers--;
+	pthread_mutex_unlock(&queue_mtx);
+	swapcontext(&fiber_context[id], return_context[id]);
+}
+
+static void *
+worker_thread(void *arg) {
+	ucontext_t scheduler_context;
+	(void)arg;
+	for (;;) {
+		int id = pop_ready_fiber();
+		if (id == -2) {
+			break;
+		}
+		if (id < 0) {
+			continue;
+		}
+		return_context[id] = &scheduler_context;
+		swapcontext(&scheduler_context, &fiber_context[id]);
+		if (!fiber_done[id]) {
+			push_ready_fiber((unsigned)id);
+		}
+	}
+	return NULL;
+}
+
+int
+main(void) {
+	void *stacks[num_fibers];
+	pthread_t threads[num_workers];
+	unsigned i;
+
+	live_fibers = num_fibers;
+	for (i = 0; i < num_fibers; i++) {
+		stacks[i] = malloc(fiber_stack_size);
+		fiber_remaining_ops[i] = ops_per_fiber;
+		getcontext(&fiber_context[i]);
+		fiber_context[i].uc_stack.ss_sp = stacks[i];
+		fiber_context[i].uc_stack.ss_size = fiber_stack_size;
+		fiber_context[i].uc_link = NULL;
+		makecontext(&fiber_context[i], (void (*)(void))fiber_run, 1, (int)i);
+		push_ready_fiber(i);
+	}
+
+	for (i = 0; i < num_workers; i++) {
+		pthread_create(&threads[i], NULL, worker_thread, NULL);
+	}
+	for (i = 0; i < num_workers; i++) {
+		pthread_join(threads[i], NULL);
+	}
+	for (i = 0; i < num_fibers; i++) {
+		free(stacks[i]);
+	}
+
+	/* Completing without a tcache-corruption crash is success. */
+	return live_fibers == 0 ? 0 : 1;
+}