From 2688047b56e6ef21d960e40281cb13774c8c17ab Mon Sep 17 00:00:00 2001 From: Slobodan Predolac Date: Mon, 29 Sep 2025 09:41:51 -0700 Subject: [PATCH] Revert "Do not dehugify when purging" This reverts commit 16c5abd1cd0a21e8f985f77d8e342c8ed91450d7. --- include/jemalloc/internal/hpa_hooks.h | 1 + src/hpa.c | 9 +++++++- src/hpa_hooks.c | 12 ++++++++-- test/unit/hpa.c | 23 +++++++++++++++++++ test/unit/hpa_vectorized_madvise.c | 11 +++++++++ .../unit/hpa_vectorized_madvise_large_batch.c | 1 + 6 files changed, 54 insertions(+), 3 deletions(-) diff --git a/include/jemalloc/internal/hpa_hooks.h b/include/jemalloc/internal/hpa_hooks.h index 5e68e349..f50ff58f 100644 --- a/include/jemalloc/internal/hpa_hooks.h +++ b/include/jemalloc/internal/hpa_hooks.h @@ -10,6 +10,7 @@ struct hpa_hooks_s { void (*unmap)(void *ptr, size_t size); void (*purge)(void *ptr, size_t size); bool (*hugify)(void *ptr, size_t size, bool sync); + void (*dehugify)(void *ptr, size_t size); void (*curtime)(nstime_t *r_time, bool first_reading); uint64_t (*ms_since)(nstime_t *r_time); bool (*vectorized_purge)(void *vec, size_t vlen, size_t nbytes); diff --git a/src/hpa.c b/src/hpa.c index d848b1ed..271b1af4 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -433,11 +433,18 @@ hpa_purge_actual_unlocked( hpa_range_accum_init(&accum, vec, len); for (size_t i = 0; i < batch_sz; ++i) { + hpdata_t *to_purge = batch[i].hp; + + /* Actually do the purging, now that the lock is dropped. */ + if (batch[i].dehugify) { + shard->central->hooks.dehugify( + hpdata_addr_get(to_purge), HUGEPAGE); + } void *purge_addr; size_t purge_size; size_t total_purged_on_one_hp = 0; while (hpdata_purge_next( - batch[i].hp, &batch[i].state, &purge_addr, &purge_size)) { + to_purge, &batch[i].state, &purge_addr, &purge_size)) { total_purged_on_one_hp += purge_size; assert(total_purged_on_one_hp <= HUGEPAGE); hpa_range_accum_add( diff --git a/src/hpa_hooks.c b/src/hpa_hooks.c index e40d30ec..14005ae0 100644 --- a/src/hpa_hooks.c +++ b/src/hpa_hooks.c @@ -8,13 +8,14 @@ static void *hpa_hooks_map(size_t size); static void hpa_hooks_unmap(void *ptr, size_t size); static void hpa_hooks_purge(void *ptr, size_t size); static bool hpa_hooks_hugify(void *ptr, size_t size, bool sync); +static void hpa_hooks_dehugify(void *ptr, size_t size); static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading); static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime); static bool hpa_hooks_vectorized_purge(void *vec, size_t vlen, size_t nbytes); const hpa_hooks_t hpa_hooks_default = {&hpa_hooks_map, &hpa_hooks_unmap, - &hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_curtime, - &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge}; + &hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_dehugify, + &hpa_hooks_curtime, &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge}; static void * hpa_hooks_map(size_t size) { @@ -60,6 +61,13 @@ hpa_hooks_hugify(void *ptr, size_t size, bool sync) { return err; } +static void +hpa_hooks_dehugify(void *ptr, size_t size) { + bool err = pages_nohuge(ptr, size); + JE_USDT(hpa_dehugify, 3, size, ptr, err); + (void)err; +} + static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading) { if (first_reading) { diff --git a/test/unit/hpa.c b/test/unit/hpa.c index d62ac762..1fed8a80 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -389,6 +389,12 @@ defer_test_hugify(void *ptr, size_t size, bool sync) { return false; } +static size_t ndefer_dehugify_calls = 0; +static void +defer_test_dehugify(void *ptr, size_t size) { + ++ndefer_dehugify_calls; +} + static nstime_t defer_curtime; static void defer_test_curtime(nstime_t *r_time, bool first_reading) { @@ -408,6 +414,7 @@ TEST_BEGIN(test_defer_time) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -446,8 +453,10 @@ TEST_BEGIN(test_defer_time) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(1, ndefer_dehugify_calls, "Should have dehugified"); expect_zu_eq(1, ndefer_purge_calls, "Should have purged"); ndefer_hugify_calls = 0; + ndefer_dehugify_calls = 0; ndefer_purge_calls = 0; /* @@ -468,6 +477,7 @@ TEST_BEGIN(test_defer_time) { nstime_init2(&defer_curtime, 22, 0); hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(1, ndefer_hugify_calls, "Failed to hugify"); + expect_zu_eq(0, ndefer_dehugify_calls, "Unexpected dehugify"); expect_zu_eq(0, ndefer_purge_calls, "Unexpected purge"); ndefer_hugify_calls = 0; @@ -514,6 +524,7 @@ TEST_BEGIN(test_no_min_purge_interval) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -540,6 +551,7 @@ TEST_BEGIN(test_no_min_purge_interval) { * we have dirty pages. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); expect_zu_eq(1, ndefer_purge_calls, "Expect purge"); ndefer_purge_calls = 0; @@ -555,6 +567,7 @@ TEST_BEGIN(test_min_purge_interval) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -580,6 +593,7 @@ TEST_BEGIN(test_min_purge_interval) { * opt.min_purge_interval_ms didn't pass yet. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); expect_zu_eq(0, ndefer_purge_calls, "Purged too early"); /* Minumum purge interval is set to 5 seconds in options. */ @@ -588,6 +602,7 @@ TEST_BEGIN(test_min_purge_interval) { /* Now we should purge, but nothing else. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); expect_zu_eq(1, ndefer_purge_calls, "Expect purge"); ndefer_purge_calls = 0; @@ -603,6 +618,7 @@ TEST_BEGIN(test_purge) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -632,6 +648,7 @@ TEST_BEGIN(test_purge) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* * Expect only 2 purges, because opt.dirty_mult is set to 0.25 and we still * have 5 active hugepages (1 / 5 = 0.2 < 0.25). @@ -648,6 +665,7 @@ TEST_BEGIN(test_purge) { */ expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification"); ndefer_hugify_calls = 0; + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* * We still have completely dirty hugepage, but we are below * opt.dirty_mult. @@ -667,6 +685,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -697,6 +716,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* * Expect only one purge call, because opts.experimental_max_purge_nhp * is set to 1. @@ -709,6 +729,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification"); ndefer_hugify_calls = 0; + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* We still above the limit for dirty pages. */ expect_zu_eq(1, ndefer_purge_calls, "Expect purge"); ndefer_purge_calls = 0; @@ -717,6 +738,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* Finally, we are below the limit, no purges are expected. */ expect_zu_eq(0, ndefer_purge_calls, "Purged too early"); @@ -732,6 +754,7 @@ TEST_BEGIN(test_vectorized_opt_eq_zero) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; diff --git a/test/unit/hpa_vectorized_madvise.c b/test/unit/hpa_vectorized_madvise.c index c2aa3b58..8df54d06 100644 --- a/test/unit/hpa_vectorized_madvise.c +++ b/test/unit/hpa_vectorized_madvise.c @@ -123,6 +123,12 @@ defer_test_hugify(void *ptr, size_t size, bool sync) { return false; } +static size_t ndefer_dehugify_calls = 0; +static void +defer_test_dehugify(void *ptr, size_t size) { + ++ndefer_dehugify_calls; +} + static nstime_t defer_curtime; static void defer_test_curtime(nstime_t *r_time, bool first_reading) { @@ -142,6 +148,7 @@ TEST_BEGIN(test_vectorized_failure_fallback) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge_fail; @@ -181,6 +188,7 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -223,6 +231,7 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) { * we have dirty pages. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* We purge from 2 huge pages, each one 3 dirty continous segments. * For opt_process_madvise_max_batch = 2, that is @@ -250,6 +259,7 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -286,6 +296,7 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) { * we have dirty pages. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); + expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* We have page batch size = 1. * we have 5 * HP active pages, 3 * HP dirty pages diff --git a/test/unit/hpa_vectorized_madvise_large_batch.c b/test/unit/hpa_vectorized_madvise_large_batch.c index c974500c..a5766620 100644 --- a/test/unit/hpa_vectorized_madvise_large_batch.c +++ b/test/unit/hpa_vectorized_madvise_large_batch.c @@ -140,6 +140,7 @@ TEST_BEGIN(test_vectorized_purge) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; + hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge;