diff --git a/include/jemalloc/internal/arena_externs.h b/include/jemalloc/internal/arena_externs.h index 8dd5b015..91fed258 100644 --- a/include/jemalloc/internal/arena_externs.h +++ b/include/jemalloc/internal/arena_externs.h @@ -106,7 +106,7 @@ unsigned arena_nthreads_get(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, const arena_config_t *config); -bool arena_init_huge(arena_t *a0); +bool arena_init_huge(tsdn_t *tsdn, arena_t *a0); arena_t *arena_choose_huge(tsd_t *tsd); bin_t *arena_bin_choose(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned *binshard); diff --git a/src/arena.c b/src/arena.c index 84d4e14c..0a0c97ef 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1904,7 +1904,7 @@ arena_choose_huge(tsd_t *tsd) { } bool -arena_init_huge(arena_t *a0) { +arena_init_huge(tsdn_t *tsdn, arena_t *a0) { bool huge_enabled; assert(huge_arena_ind == 0); @@ -1922,13 +1922,18 @@ arena_init_huge(arena_t *a0) { /* a0 init happened before malloc_conf_init. */ atomic_store_zu(&a0->pa_shard.pac.oversize_threshold, oversize_threshold, ATOMIC_RELAXED); - /* Initialize huge arena THP settings for PAC. */ + /* Initialize huge_arena_pac_thp fields. */ + base_t *b0 = a0->base; + /* Make sure that b0 thp auto-switch won't happen concurrently here. */ + malloc_mutex_lock(tsdn, &b0->mtx); (&huge_arena_pac_thp)->thp_madvise = opt_huge_arena_pac_thp && metadata_thp_enabled() && (opt_thp == thp_mode_default) && (init_system_thp_mode == thp_mode_default); + (&huge_arena_pac_thp)->auto_thp_switched = b0->auto_thp_switched; malloc_mutex_init(&(&huge_arena_pac_thp)->lock, "pac_thp", WITNESS_RANK_LEAF, malloc_mutex_rank_exclusive); edata_list_active_init(&(&huge_arena_pac_thp)->thp_lazy_list); + malloc_mutex_unlock(tsdn, &b0->mtx); huge_enabled = true; } diff --git a/src/base.c b/src/base.c index 13367697..52f3d1d3 100644 --- a/src/base.c +++ b/src/base.c @@ -153,17 +153,19 @@ base_get_num_blocks(base_t *base, bool with_new_block) { static void huge_arena_auto_thp_switch(tsdn_t *tsdn, pac_thp_t *pac_thp) { assert(opt_huge_arena_pac_thp); - assert(!pac_thp->auto_thp_switched); - - arena_t *huge_arena; - if (huge_arena_ind == 0 || (huge_arena = arena_get(tsdn, huge_arena_ind, - false)) == NULL) { - /* Huge arena hasn't been init yet, simply turn the switch on. */ - pac_thp->auto_thp_switched = true; +#ifdef JEMALLOC_JET + if (pac_thp->auto_thp_switched) { return; } +#else + /* + * The switch should be turned on only once when the b0 auto thp switch is + * turned on, unless it's a unit test where b0 gets deleted and then + * recreated. + */ + assert(!pac_thp->auto_thp_switched); +#endif - assert(huge_arena != NULL); edata_list_active_t *pending_list; malloc_mutex_lock(tsdn, &pac_thp->lock); pending_list = &pac_thp->thp_lazy_list; @@ -221,9 +223,19 @@ base_auto_thp_switch(tsdn_t *tsdn, base_t *base) { /* Handle the THP auto switch for the huge arena. */ if (!huge_arena_pac_thp.thp_madvise || base_ind_get(base) != 0) { - /* Only b0 metadata auto thp switch do the trigger. */ + /* + * The huge arena THP auto-switch is triggered only by b0 switch, + * provided that the huge arena is initialized. If b0 switch is enabled + * before huge arena is ready, the huge arena switch will be enabled + * during huge_arena_pac_thp initialization. + */ return; } + /* + * thp_madvise above is by default false and set in arena_init_huge() with + * b0 mtx held. So if we reach here, it means the entire huge_arena_pac_thp + * is initialized and we can safely switch the THP. + */ malloc_mutex_unlock(tsdn, &base->mtx); huge_arena_auto_thp_switch(tsdn, &huge_arena_pac_thp); malloc_mutex_lock(tsdn, &base->mtx); diff --git a/src/jemalloc.c b/src/jemalloc.c index 4939d954..d7b46d6c 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2097,7 +2097,7 @@ percpu_arena_as_initialized(percpu_arena_mode_t mode) { } static bool -malloc_init_narenas(void) { +malloc_init_narenas(tsdn_t *tsdn) { assert(ncpus > 0); if (opt_percpu_arena != percpu_arena_disabled) { @@ -2164,7 +2164,7 @@ malloc_init_narenas(void) { narenas_auto); } narenas_total_set(narenas_auto); - if (arena_init_huge(a0)) { + if (arena_init_huge(tsdn, a0)) { narenas_total_inc(); } manual_arena_base = narenas_total_get(); @@ -2248,7 +2248,7 @@ malloc_init_hard(void) { /* Set reentrancy level to 1 during init. */ pre_reentrancy(tsd, NULL); /* Initialize narenas before prof_boot2 (for allocation). */ - if (malloc_init_narenas() + if (malloc_init_narenas(tsd_tsdn(tsd)) || background_thread_boot1(tsd_tsdn(tsd), b0get())) { UNLOCK_RETURN(tsd_tsdn(tsd), true, true) }