romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-04-14 22:51:50 +03:00
Code Issues Projects Releases Packages Wiki Activity

Do not dehugify when purging

Browse source 
Giving the advice MADV_DONTNEED to a range of virtual memory backed by
a transparent huge page already causes that range of virtual memory to
become backed by regular pages.
This commit is contained in:
lexprfuncall 2025-08-04 14:43:03 -07:00 • committed by guangli-dai
parent 965b3a1130
commit 9bd26fc9f5
6 changed files with 3 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

1
include/jemalloc/internal/hpa_hooks.h
View file

@ -10,7 +10,6 @@ struct hpa_hooks_s {
void (*unmap)(void *ptr, size_t size);
void (*purge)(void *ptr, size_t size);
bool (*hugify)(void *ptr, size_t size, bool sync);
void (*dehugify)(void *ptr, size_t size);
void (*curtime)(nstime_t *r_time, bool first_reading);
uint64_t (*ms_since)(nstime_t *r_time);
bool (*vectorized_purge)(void *vec, size_t vlen, size_t nbytes);

9
src/hpa.c
View file

@ -432,18 +432,11 @@ hpa_purge_actual_unlocked(
hpa_range_accum_init(&accum, vec, len);
for (size_t i = 0; i < batch_sz; ++i) {
hpdata_t *to_purge = batch[i].hp;
/* Actually do the purging, now that the lock is dropped. */
if (batch[i].dehugify) {
shard->central->hooks.dehugify(
hpdata_addr_get(to_purge), HUGEPAGE);
}
void *purge_addr;
size_t purge_size;
size_t total_purged_on_one_hp = 0;
while (hpdata_purge_next(
to_purge, &batch[i].state, &purge_addr, &purge_size)) {
batch[i].hp, &batch[i].state, &purge_addr, &purge_size)) {
total_purged_on_one_hp += purge_size;
assert(total_purged_on_one_hp <= HUGEPAGE);
hpa_range_accum_add(

12
src/hpa_hooks.c
View file

@ -8,14 +8,13 @@ static void *hpa_hooks_map(size_t size);
static void hpa_hooks_unmap(void *ptr, size_t size);
static void hpa_hooks_purge(void *ptr, size_t size);
static bool hpa_hooks_hugify(void *ptr, size_t size, bool sync);
static void hpa_hooks_dehugify(void *ptr, size_t size);
static void hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading);
static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime);
static bool hpa_hooks_vectorized_purge(void *vec, size_t vlen, size_t nbytes);
const hpa_hooks_t hpa_hooks_default = {&hpa_hooks_map, &hpa_hooks_unmap,
&hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_dehugify,
&hpa_hooks_curtime, &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge};
&hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_curtime,
&hpa_hooks_ms_since, &hpa_hooks_vectorized_purge};
static void *
hpa_hooks_map(size_t size) {
@ -61,13 +60,6 @@ hpa_hooks_hugify(void *ptr, size_t size, bool sync) {
return err;
}
static void
hpa_hooks_dehugify(void *ptr, size_t size) {
bool err = pages_nohuge(ptr, size);
JE_USDT(hpa_dehugify, 3, size, ptr, err);
(void)err;
}
static void
hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading) {
if (first_reading) {

23
test/unit/hpa.c
View file

@ -389,12 +389,6 @@ defer_test_hugify(void *ptr, size_t size, bool sync) {
return false;
}
static size_t ndefer_dehugify_calls = 0;
static void
defer_test_dehugify(void *ptr, size_t size) {
++ndefer_dehugify_calls;
}
static nstime_t defer_curtime;
static void
defer_test_curtime(nstime_t *r_time, bool first_reading) {
@ -414,7 +408,6 @@ TEST_BEGIN(test_defer_time) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
@ -453,10 +446,8 @@ TEST_BEGIN(test_defer_time) {
hpa_shard_do_deferred_work(tsdn, shard);
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(1, ndefer_dehugify_calls, "Should have dehugified");
expect_zu_eq(1, ndefer_purge_calls, "Should have purged");
ndefer_hugify_calls = 0;
ndefer_dehugify_calls = 0;
ndefer_purge_calls = 0;
/*
@ -477,7 +468,6 @@ TEST_BEGIN(test_defer_time) {
nstime_init2(&defer_curtime, 22, 0);
hpa_shard_do_deferred_work(tsdn, shard);
expect_zu_eq(1, ndefer_hugify_calls, "Failed to hugify");
expect_zu_eq(0, ndefer_dehugify_calls, "Unexpected dehugify");
expect_zu_eq(0, ndefer_purge_calls, "Unexpected purge");
ndefer_hugify_calls = 0;
@ -524,7 +514,6 @@ TEST_BEGIN(test_no_min_purge_interval) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
@ -551,7 +540,6 @@ TEST_BEGIN(test_no_min_purge_interval) {
* we have dirty pages.
*/
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
expect_zu_eq(1, ndefer_purge_calls, "Expect purge");
ndefer_purge_calls = 0;
@ -567,7 +555,6 @@ TEST_BEGIN(test_min_purge_interval) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
@ -593,7 +580,6 @@ TEST_BEGIN(test_min_purge_interval) {
* opt.min_purge_interval_ms didn't pass yet.
*/
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
expect_zu_eq(0, ndefer_purge_calls, "Purged too early");
/* Minumum purge interval is set to 5 seconds in options. */
@ -602,7 +588,6 @@ TEST_BEGIN(test_min_purge_interval) {
/* Now we should purge, but nothing else. */
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
expect_zu_eq(1, ndefer_purge_calls, "Expect purge");
ndefer_purge_calls = 0;
@ -618,7 +603,6 @@ TEST_BEGIN(test_purge) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
@ -648,7 +632,6 @@ TEST_BEGIN(test_purge) {
hpa_shard_do_deferred_work(tsdn, shard);
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
/*
* Expect only 2 purges, because opt.dirty_mult is set to 0.25 and we still
* have 5 active hugepages (1 / 5 = 0.2 < 0.25).
@ -665,7 +648,6 @@ TEST_BEGIN(test_purge) {
*/
expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification");
ndefer_hugify_calls = 0;
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
/*
* We still have completely dirty hugepage, but we are below
* opt.dirty_mult.
@ -685,7 +667,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
@ -716,7 +697,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
hpa_shard_do_deferred_work(tsdn, shard);
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
/*
* Expect only one purge call, because opts.experimental_max_purge_nhp
* is set to 1.
@ -729,7 +709,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification");
ndefer_hugify_calls = 0;
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
/* We still above the limit for dirty pages. */
expect_zu_eq(1, ndefer_purge_calls, "Expect purge");
ndefer_purge_calls = 0;
@ -738,7 +717,6 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
hpa_shard_do_deferred_work(tsdn, shard);
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
/* Finally, we are below the limit, no purges are expected. */
expect_zu_eq(0, ndefer_purge_calls, "Purged too early");
@ -754,7 +732,6 @@ TEST_BEGIN(test_vectorized_opt_eq_zero) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;

11
test/unit/hpa_vectorized_madvise.c
View file

@ -123,12 +123,6 @@ defer_test_hugify(void *ptr, size_t size, bool sync) {
return false;
}
static size_t ndefer_dehugify_calls = 0;
static void
defer_test_dehugify(void *ptr, size_t size) {
++ndefer_dehugify_calls;
}
static nstime_t defer_curtime;
static void
defer_test_curtime(nstime_t *r_time, bool first_reading) {
@ -148,7 +142,6 @@ TEST_BEGIN(test_vectorized_failure_fallback) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge_fail;
@ -188,7 +181,6 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
@ -231,7 +223,6 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) {
* we have dirty pages.
*/
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
/* We purge from 2 huge pages, each one 3 dirty continous segments.
* For opt_process_madvise_max_batch = 2, that is
@ -259,7 +250,6 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;
@ -296,7 +286,6 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) {
* we have dirty pages.
*/
expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
/* We have page batch size = 1.
* we have 5 * HP active pages, 3 * HP dirty pages

1
test/unit/hpa_vectorized_madvise_large_batch.c
View file

@ -140,7 +140,6 @@ TEST_BEGIN(test_vectorized_purge) {
hooks.unmap = &defer_test_unmap;
hooks.purge = &defer_test_purge;
hooks.hugify = &defer_test_hugify;
hooks.dehugify = &defer_test_dehugify;
hooks.curtime = &defer_test_curtime;
hooks.ms_since = &defer_test_ms_since;
hooks.vectorized_purge = &defer_vectorized_purge;