Revert "Do not dehugify when purging"

This reverts commit 16c5abd1cd.

include/jemalloc/internal/hpa_hooks.h

@@ -10,6 +10,7 @@ struct hpa_hooks_s {
 	void (*unmap)(void *ptr, size_t size);
 	void (*purge)(void *ptr, size_t size);
 	bool (*hugify)(void *ptr, size_t size, bool sync);
+	void (*dehugify)(void *ptr, size_t size);
 	void (*curtime)(nstime_t *r_time, bool first_reading);
 	uint64_t (*ms_since)(nstime_t *r_time);
 	bool (*vectorized_purge)(void *vec, size_t vlen, size_t nbytes);

src/hpa.c

@@ -433,11 +433,18 @@ hpa_purge_actual_unlocked(
 	hpa_range_accum_init(&accum, vec, len);
 
 	for (size_t i = 0; i < batch_sz; ++i) {
+		hpdata_t *to_purge = batch[i].hp;
+
+		/* Actually do the purging, now that the lock is dropped. */
+		if (batch[i].dehugify) {
+			shard->central->hooks.dehugify(
+			    hpdata_addr_get(to_purge), HUGEPAGE);
+		}
 		void  *purge_addr;
 		size_t purge_size;
 		size_t total_purged_on_one_hp = 0;
 		while (hpdata_purge_next(
-		    batch[i].hp, &batch[i].state, &purge_addr, &purge_size)) {
+		    to_purge, &batch[i].state, &purge_addr, &purge_size)) {
 			total_purged_on_one_hp += purge_size;
 			assert(total_purged_on_one_hp <= HUGEPAGE);
 			hpa_range_accum_add(

src/hpa_hooks.c

@@ -8,13 +8,14 @@ static void    *hpa_hooks_map(size_t size);
 static void     hpa_hooks_unmap(void *ptr, size_t size);
 static void     hpa_hooks_purge(void *ptr, size_t size);
 static bool     hpa_hooks_hugify(void *ptr, size_t size, bool sync);
+static void     hpa_hooks_dehugify(void *ptr, size_t size);
 static void     hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading);
 static uint64_t hpa_hooks_ms_since(nstime_t *past_nstime);
 static bool hpa_hooks_vectorized_purge(void *vec, size_t vlen, size_t nbytes);
 
 const hpa_hooks_t hpa_hooks_default = {&hpa_hooks_map, &hpa_hooks_unmap,
-    &hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_curtime,
-    &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge};
+    &hpa_hooks_purge, &hpa_hooks_hugify, &hpa_hooks_dehugify,
+    &hpa_hooks_curtime, &hpa_hooks_ms_since, &hpa_hooks_vectorized_purge};
 
 static void *
 hpa_hooks_map(size_t size) {
@@ -60,6 +61,13 @@ hpa_hooks_hugify(void *ptr, size_t size, bool sync) {
 	return err;
 }
 
+static void
+hpa_hooks_dehugify(void *ptr, size_t size) {
+	bool err = pages_nohuge(ptr, size);
+	JE_USDT(hpa_dehugify, 3, size, ptr, err);
+	(void)err;
+}
+
 static void
 hpa_hooks_curtime(nstime_t *r_nstime, bool first_reading) {
 	if (first_reading) {

test/unit/hpa.c

@@ -389,6 +389,12 @@ defer_test_hugify(void *ptr, size_t size, bool sync) {
 	return false;
 }
 
+static size_t ndefer_dehugify_calls = 0;
+static void
+defer_test_dehugify(void *ptr, size_t size) {
+	++ndefer_dehugify_calls;
+}
+
 static nstime_t defer_curtime;
 static void
 defer_test_curtime(nstime_t *r_time, bool first_reading) {
@@ -408,6 +414,7 @@ TEST_BEGIN(test_defer_time) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;
@@ -446,8 +453,10 @@ TEST_BEGIN(test_defer_time) {
 	hpa_shard_do_deferred_work(tsdn, shard);
 
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(1, ndefer_dehugify_calls, "Should have dehugified");
 	expect_zu_eq(1, ndefer_purge_calls, "Should have purged");
 	ndefer_hugify_calls = 0;
+	ndefer_dehugify_calls = 0;
 	ndefer_purge_calls = 0;
 
 	/*
@@ -468,6 +477,7 @@ TEST_BEGIN(test_defer_time) {
 	nstime_init2(&defer_curtime, 22, 0);
 	hpa_shard_do_deferred_work(tsdn, shard);
 	expect_zu_eq(1, ndefer_hugify_calls, "Failed to hugify");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Unexpected dehugify");
 	expect_zu_eq(0, ndefer_purge_calls, "Unexpected purge");
 	ndefer_hugify_calls = 0;
 
@@ -514,6 +524,7 @@ TEST_BEGIN(test_no_min_purge_interval) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;
@@ -540,6 +551,7 @@ TEST_BEGIN(test_no_min_purge_interval) {
 	 * we have dirty pages.
 	 */
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	expect_zu_eq(1, ndefer_purge_calls, "Expect purge");
 	ndefer_purge_calls = 0;
 
@@ -555,6 +567,7 @@ TEST_BEGIN(test_min_purge_interval) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;
@@ -580,6 +593,7 @@ TEST_BEGIN(test_min_purge_interval) {
 	 * opt.min_purge_interval_ms didn't pass yet.
 	 */
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	expect_zu_eq(0, ndefer_purge_calls, "Purged too early");
 
 	/* Minumum purge interval is set to 5 seconds in options. */
@@ -588,6 +602,7 @@ TEST_BEGIN(test_min_purge_interval) {
 
 	/* Now we should purge, but nothing else. */
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	expect_zu_eq(1, ndefer_purge_calls, "Expect purge");
 	ndefer_purge_calls = 0;
 
@@ -603,6 +618,7 @@ TEST_BEGIN(test_purge) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;
@@ -632,6 +648,7 @@ TEST_BEGIN(test_purge) {
 	hpa_shard_do_deferred_work(tsdn, shard);
 
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	/*
 	 * Expect only 2 purges, because opt.dirty_mult is set to 0.25 and we still
 	 * have 5 active hugepages (1 / 5 = 0.2 < 0.25).
@@ -648,6 +665,7 @@ TEST_BEGIN(test_purge) {
 	 */
 	expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification");
 	ndefer_hugify_calls = 0;
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	/*
 	 * We still have completely dirty hugepage, but we are below
 	 * opt.dirty_mult.
@@ -667,6 +685,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;
@@ -697,6 +716,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
 	hpa_shard_do_deferred_work(tsdn, shard);
 
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	/*
 	 * Expect only one purge call, because opts.experimental_max_purge_nhp
 	 * is set to 1.
@@ -709,6 +729,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
 
 	expect_zu_eq(5, ndefer_hugify_calls, "Expect hugification");
 	ndefer_hugify_calls = 0;
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	/* We still above the limit for dirty pages. */
 	expect_zu_eq(1, ndefer_purge_calls, "Expect purge");
 	ndefer_purge_calls = 0;
@@ -717,6 +738,7 @@ TEST_BEGIN(test_experimental_max_purge_nhp) {
 	hpa_shard_do_deferred_work(tsdn, shard);
 
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 	/* Finally, we are below the limit, no purges are expected. */
 	expect_zu_eq(0, ndefer_purge_calls, "Purged too early");
 
@@ -732,6 +754,7 @@ TEST_BEGIN(test_vectorized_opt_eq_zero) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;

test/unit/hpa_vectorized_madvise.c

@@ -123,6 +123,12 @@ defer_test_hugify(void *ptr, size_t size, bool sync) {
 	return false;
 }
 
+static size_t ndefer_dehugify_calls = 0;
+static void
+defer_test_dehugify(void *ptr, size_t size) {
+	++ndefer_dehugify_calls;
+}
+
 static nstime_t defer_curtime;
 static void
 defer_test_curtime(nstime_t *r_time, bool first_reading) {
@@ -142,6 +148,7 @@ TEST_BEGIN(test_vectorized_failure_fallback) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge_fail;
@@ -181,6 +188,7 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;
@@ -223,6 +231,7 @@ TEST_BEGIN(test_more_regions_purged_from_one_page) {
 	 * we have dirty pages.
 	 */
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 
 	/* We purge from 2 huge pages, each one 3 dirty continous segments.
 	 * For opt_process_madvise_max_batch = 2, that is
@@ -250,6 +259,7 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;
@@ -286,6 +296,7 @@ TEST_BEGIN(test_more_pages_than_batch_page_size) {
 	 * we have dirty pages.
 	 */
 	expect_zu_eq(0, ndefer_hugify_calls, "Hugified too early");
+	expect_zu_eq(0, ndefer_dehugify_calls, "Dehugified too early");
 
 	/* We have page batch size = 1.
 	 * we have 5 * HP active pages, 3 * HP dirty pages

test/unit/hpa_vectorized_madvise_large_batch.c

@@ -140,6 +140,7 @@ TEST_BEGIN(test_vectorized_purge) {
 	hooks.unmap = &defer_test_unmap;
 	hooks.purge = &defer_test_purge;
 	hooks.hugify = &defer_test_hugify;
+	hooks.dehugify = &defer_test_dehugify;
 	hooks.curtime = &defer_test_curtime;
 	hooks.ms_since = &defer_test_ms_since;
 	hooks.vectorized_purge = &defer_vectorized_purge;