From 9bd26fc9f5a74c93b4c78405224063fc3d2cd0ba Mon Sep 17 00:00:00 2001 From: lexprfuncall <5360361+lexprfuncall@users.noreply.github.com> Date: Mon, 4 Aug 2025 14:43:03 -0700 Subject: [PATCH] Do not dehugify when purging Giving the advice MADV_DONTNEED to a range of virtual memory backed by a transparent huge page already causes that range of virtual memory to become backed by regular pages. --- include/jemalloc/internal/hpa_hooks.h | 1 - src/hpa.c | 9 +------- src/hpa_hooks.c | 12 ++-------- test/unit/hpa.c | 23 ------------------- test/unit/hpa_vectorized_madvise.c | 11 --------- .../unit/hpa_vectorized_madvise_large_batch.c | 1 - 6 files changed, 3 insertions(+), 54 deletions(-) diff --git a/include/jemalloc/internal/hpa_hooks.h b/include/jemalloc/internal/hpa_hooks.h index f50ff58f..5e68e349 100644 --- a/include/jemalloc/internal/hpa_hooks.h +++ b/include/jemalloc/internal/hpa_hooks.h @@ -10,7 +10,6 @@ struct hpa_hooks_s { void (*unmap)(void *ptr, size_t size); void (*purge)(void *ptr, size_t size); bool (*hugify)(void *ptr, size_t size, bool sync); - void (*dehugify)(void *ptr, size_t size); void (*curtime)(nstime_t *r_time, bool first_reading); uint64_t (*ms_since)(nstime_t *r_time); bool (*vectorized_purge)(void *vec, size_t vlen, size_t nbytes); diff --git a/src/hpa.c b/src/hpa.c index 4c0f4e36..e297e411 100644 --- a/src/hpa.c +++ b/src/hpa.c @@ -432,18 +432,11 @@ hpa_purge_actual_unlocked( hpa_range_accum_init(&accum, vec, len); for (size_t i = 0; i < batch_sz; ++i) { - hpdata_t *to_purge = batch[i].hp; - - /* Actually do the purging, now that the lock is dropped. */ - if (batch[i].dehugify) { - shard->central->hooks.dehugify( - hpdata_addr_get(to_purge), HUGEPAGE); - } void *purge_addr; size_t purge_size; size_t total_purged_on_one_hp = 0; while (hpdata_purge_next( - to_purge, &batch[i].state, &purge_addr, &purge_size)) { + batch[i].hp, &batch[i].state, &purge_addr, &purge_size)) { total_purged_on_one_hp += purge_size; assert(total_purged_on_one_hp <= HUGEPAGE); hpa_range_accum_add( diff --git a/src/hpa_hooks.c b/src/hpa_hooks.c index 14005ae0..e40d30ec 100644 --- a/src/hpa_hooks.c +++ b/src/hpa_hooks.c @@ -8,14 +8,13 @@ static void *hpa_hooks_map(size_t size); static void hpa_hooks_unmap(void *ptr, size_t size); static void hpa_hooks_purge(void *ptr, size_t size); static bool hpa_hooks_hugify(void *ptr, size_t size, bool sync); -static void hpa_hooks_dehugify(void *ptr, size_t size); static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading); static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime); static bool hpa_hooks_vectorized_purge(void *vec, size_t vlen, size_t nbytes); const hpa_hooks_t hpa_hooks_default = {&hpa_hooks_map, &hpa_hooks_unmap, - &hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_dehugify, - &hpa_hooks_curtime, &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge}; + &hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_curtime, + &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge}; static void * hpa_hooks_map(size_t size) { @@ -61,13 +60,6 @@ hpa_hooks_hugify(void *ptr, size_t size, bool sync) { return err; } -static void -hpa_hooks_dehugify(void *ptr, size_t size) { - bool err = pages_nohuge(ptr, size); - JE_USDT(hpa_dehugify, 3, size, ptr, err); - (void)err; -} - static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading) { if (first_reading) { diff --git a/test/unit/hpa.c b/test/unit/hpa.c index 1fed8a80..d62ac762 100644 --- a/test/unit/hpa.c +++ b/test/unit/hpa.c @@ -389,12 +389,6 @@ defer_test_hugify(void *ptr, size_t size, bool sync) { return false; } -static size_t ndefer_dehugify_calls = 0; -static void -defer_test_dehugify(void *ptr, size_t size) { - ++ndefer_dehugify_calls; -} - static nstime_t defer_curtime; static void defer_test_curtime(nstime_t *r_time, bool first_reading) { @@ -414,7 +408,6 @@ TEST_BEGIN(test_defer_time) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -453,10 +446,8 @@ TEST_BEGIN(test_defer_time) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(1, ndefer_dehugify_calls, "Should have dehugified"); expect_zu_eq(1, ndefer_purge_calls, "Should have purged"); ndefer_hugify_calls = 0; - ndefer_dehugify_calls = 0; ndefer_purge_calls = 0; /* @@ -477,7 +468,6 @@ TEST_BEGIN(test_defer_time) { nstime_init2(&defer_curtime, 22, 0); hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(1, ndefer_hugify_calls, "Failed to hugify"); - expect_zu_eq(0, ndefer_dehugify_calls, "Unexpected dehugify"); expect_zu_eq(0, ndefer_purge_calls, "Unexpected purge"); ndefer_hugify_calls = 0; @@ -524,7 +514,6 @@ TEST_BEGIN(test_no_min_purge_interval) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -551,7 +540,6 @@ TEST_BEGIN(test_no_min_purge_interval) { * we have dirty pages. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); expect_zu_eq(1, ndefer_purge_calls, "Expect purge"); ndefer_purge_calls = 0; @@ -567,7 +555,6 @@ TEST_BEGIN(test_min_purge_interval) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -593,7 +580,6 @@ TEST_BEGIN(test_min_purge_interval) { * opt.min_purge_interval_ms didn't pass yet. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); expect_zu_eq(0, ndefer_purge_calls, "Purged too early"); /* Minumum purge interval is set to 5 seconds in options. */ @@ -602,7 +588,6 @@ TEST_BEGIN(test_min_purge_interval) { /* Now we should purge, but nothing else. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); expect_zu_eq(1, ndefer_purge_calls, "Expect purge"); ndefer_purge_calls = 0; @@ -618,7 +603,6 @@ TEST_BEGIN(test_purge) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -648,7 +632,6 @@ TEST_BEGIN(test_purge) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* * Expect only 2 purges, because opt.dirty_mult is set to 0.25 and we still * have 5 active hugepages (1 / 5 = 0.2 < 0.25). @@ -665,7 +648,6 @@ TEST_BEGIN(test_purge) { */ expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification"); ndefer_hugify_calls = 0; - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* * We still have completely dirty hugepage, but we are below * opt.dirty_mult. @@ -685,7 +667,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -716,7 +697,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* * Expect only one purge call, because opts.experimental_max_purge_nhp * is set to 1. @@ -729,7 +709,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification"); ndefer_hugify_calls = 0; - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* We still above the limit for dirty pages. */ expect_zu_eq(1, ndefer_purge_calls, "Expect purge"); ndefer_purge_calls = 0; @@ -738,7 +717,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) { hpa_shard_do_deferred_work(tsdn, shard); expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* Finally, we are below the limit, no purges are expected. */ expect_zu_eq(0, ndefer_purge_calls, "Purged too early"); @@ -754,7 +732,6 @@ TEST_BEGIN(test_vectorized_opt_eq_zero) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; diff --git a/test/unit/hpa_vectorized_madvise.c b/test/unit/hpa_vectorized_madvise.c index 8df54d06..c2aa3b58 100644 --- a/test/unit/hpa_vectorized_madvise.c +++ b/test/unit/hpa_vectorized_madvise.c @@ -123,12 +123,6 @@ defer_test_hugify(void *ptr, size_t size, bool sync) { return false; } -static size_t ndefer_dehugify_calls = 0; -static void -defer_test_dehugify(void *ptr, size_t size) { - ++ndefer_dehugify_calls; -} - static nstime_t defer_curtime; static void defer_test_curtime(nstime_t *r_time, bool first_reading) { @@ -148,7 +142,6 @@ TEST_BEGIN(test_vectorized_failure_fallback) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge_fail; @@ -188,7 +181,6 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -231,7 +223,6 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) { * we have dirty pages. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* We purge from 2 huge pages, each one 3 dirty continous segments. * For opt_process_madvise_max_batch = 2, that is @@ -259,7 +250,6 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge; @@ -296,7 +286,6 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) { * we have dirty pages. */ expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early"); - expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early"); /* We have page batch size = 1. * we have 5 * HP active pages, 3 * HP dirty pages diff --git a/test/unit/hpa_vectorized_madvise_large_batch.c b/test/unit/hpa_vectorized_madvise_large_batch.c index a5766620..c974500c 100644 --- a/test/unit/hpa_vectorized_madvise_large_batch.c +++ b/test/unit/hpa_vectorized_madvise_large_batch.c @@ -140,7 +140,6 @@ TEST_BEGIN(test_vectorized_purge) { hooks.unmap = &defer_test_unmap; hooks.purge = &defer_test_purge; hooks.hugify = &defer_test_hugify; - hooks.dehugify = &defer_test_dehugify; hooks.curtime = &defer_test_curtime; hooks.ms_since = &defer_test_ms_since; hooks.vectorized_purge = &defer_vectorized_purge;