HPA: Track pending purges/hugifies in the psset.

This finishes the refactoring of the HPA/psset interactions the past few commits
have been building towards.

Rather than the HPA removing and then reinserting hpdatas, it simply begins
updates and ends them.  These updates can set flags on the hpdata that prevent
it from being returned for certain types of requests.  For example, it can call
hpdata_alloc_allowed_set(hpdata, false) during an update, at which point the
given hpdata will no longer be returned for psset_pick_alloc requests.

This has various of benefits:
- It maintains stats correctness during purges and hugifies.
- It allows simpler and more explicit concurrency control for the various
  special cases (e.g. allocations are disallowed during purge, but not during
  hugify).
- It lets allocations and deallocations avoid disturbing the purging and
  hugification orderings.  If an hpdata "loses its place" in one of the queues
  just do to an alloc / dalloc, it can result in pathological edge cases where
  very hot, very full hugepages never get hugified  (and cold extents on the
  same hugepage as hot ones never get purged).

The key benefit though is that tracking hpdatas to be purged / hugified in a
principled way will let us do delayed purging and hugification.  Eventually this
will let us move these operations to background threads, but in the short term
the benefit is that it will let us have global purging policies (e.g. purge when
the entire arena has too many dirty pages, rather than any particular hugepage).

src/psset.c

@@ -15,7 +15,9 @@ psset_init(psset_t *psset) {
 	}
 	bitmap_init(psset->bitmap, &psset_bitmap_info, /* fill */ true);
 	memset(&psset->stats, 0, sizeof(psset->stats));
-	hpdata_empty_list_init(&psset->empty_slabs);
+	hpdata_empty_list_init(&psset->empty);
+	hpdata_purge_list_init(&psset->to_purge);
+	hpdata_hugify_list_init(&psset->to_hugify);
 }
 
 static void
@@ -85,25 +87,56 @@ psset_hpdata_heap_insert(psset_t *psset, pszind_t pind, hpdata_t *ps) {
 	hpdata_age_heap_insert(&psset->pageslabs[pind], ps);
 }
 
-/*
- * Insert ps into the data structures we use to track allocation stats and pick
- * the pageslabs for new allocations.
- *
- * In particular, this does *not* remove ps from any hugification / purging
- * queues it may be in.
- */
 static void
-psset_do_alloc_tracking_insert(psset_t *psset, hpdata_t *ps) {
+psset_stats_insert(psset_t* psset, hpdata_t *ps) {
 	if (hpdata_empty(ps)) {
 		psset_bin_stats_insert(psset->stats.empty_slabs, ps);
+	} else if (hpdata_full(ps)) {
+		psset_bin_stats_insert(psset->stats.full_slabs, ps);
+	} else {
+		size_t longest_free_range = hpdata_longest_free_range_get(ps);
+
+		pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
+		    longest_free_range << LG_PAGE));
+		assert(pind < PSSET_NPSIZES);
+
+		psset_bin_stats_insert(psset->stats.nonfull_slabs[pind], ps);
+	}
+}
+
+static void
+psset_stats_remove(psset_t *psset, hpdata_t *ps) {
+	if (hpdata_empty(ps)) {
+		psset_bin_stats_remove(psset->stats.empty_slabs, ps);
+	} else if (hpdata_full(ps)) {
+		psset_bin_stats_remove(psset->stats.full_slabs, ps);
+	} else {
+		size_t longest_free_range = hpdata_longest_free_range_get(ps);
+
+		pszind_t pind = sz_psz2ind(sz_psz_quantize_floor(
+		    longest_free_range << LG_PAGE));
+		assert(pind < PSSET_NPSIZES);
+
+		psset_bin_stats_remove(psset->stats.nonfull_slabs[pind], ps);
+	}
+}
+
+/*
+ * Put ps into some container so that it can be found during future allocation
+ * requests.
+ */
+static void
+psset_alloc_container_insert(psset_t *psset, hpdata_t *ps) {
+	assert(!hpdata_in_psset_alloc_container_get(ps));
+	hpdata_in_psset_alloc_container_set(ps, true);
+	if (hpdata_empty(ps)) {
 		/*
 		 * This prepend, paired with popping the head in psset_fit,
 		 * means we implement LIFO ordering for the empty slabs set,
 		 * which seems reasonable.
 		 */
-		hpdata_empty_list_prepend(&psset->empty_slabs, ps);
+		hpdata_empty_list_prepend(&psset->empty, ps);
 	} else if (hpdata_full(ps)) {
-		psset_bin_stats_insert(psset->stats.full_slabs, ps);
 		/*
 		 * We don't need to keep track of the full slabs; we're never
 		 * going to return them from a psset_pick_alloc call.
@@ -115,23 +148,20 @@ psset_do_alloc_tracking_insert(psset_t *psset, hpdata_t *ps) {
 		    longest_free_range << LG_PAGE));
 		assert(pind < PSSET_NPSIZES);
 
-		psset_bin_stats_insert(psset->stats.nonfull_slabs[pind], ps);
 		psset_hpdata_heap_insert(psset, pind, ps);
 	}
 }
 
 /* Remove ps from those collections. */
 static void
-psset_do_alloc_tracking_remove(psset_t *psset, hpdata_t *ps) {
+psset_alloc_container_remove(psset_t *psset, hpdata_t *ps) {
+	assert(hpdata_in_psset_alloc_container_get(ps));
+	hpdata_in_psset_alloc_container_set(ps, false);
+
 	if (hpdata_empty(ps)) {
-		psset_bin_stats_remove(psset->stats.empty_slabs, ps);
-		hpdata_empty_list_remove(&psset->empty_slabs, ps);
+		hpdata_empty_list_remove(&psset->empty, ps);
 	} else if (hpdata_full(ps)) {
-		/*
-		 * We don't need to maintain an explicit container of full
-		 * pageslabs anywhere, but we do have to update stats.
-		 */
-		psset_bin_stats_remove(psset->stats.full_slabs, ps);
+		/* Same as above -- do nothing in this case. */
 	} else {
 		size_t longest_free_range = hpdata_longest_free_range_get(ps);
 
@@ -139,7 +169,6 @@ psset_do_alloc_tracking_remove(psset_t *psset, hpdata_t *ps) {
 		    longest_free_range << LG_PAGE));
 		assert(pind < PSSET_NPSIZES);
 
-		psset_bin_stats_remove(psset->stats.nonfull_slabs[pind], ps);
 		psset_hpdata_heap_remove(psset, pind, ps);
 	}
 }
@@ -149,7 +178,21 @@ psset_update_begin(psset_t *psset, hpdata_t *ps) {
 	hpdata_assert_consistent(ps);
 	assert(hpdata_in_psset_get(ps));
 	hpdata_updating_set(ps, true);
-	psset_do_alloc_tracking_remove(psset, ps);
+	psset_stats_remove(psset, ps);
+	if (hpdata_in_psset_alloc_container_get(ps)) {
+		/*
+		 * Some metadata updates can break alloc container invariants
+		 * (e.g. the longest free range determines the hpdata_heap_t the
+		 * pageslab lives in).
+		 */
+		assert(hpdata_alloc_allowed_get(ps));
+		psset_alloc_container_remove(psset, ps);
+	}
+	/*
+	 * We don't update presence in the purge list or hugify list; we try to
+	 * keep those FIFO, even in the presence of other metadata updates.
+	 * We'll update presence at the end of the metadata update if necessary.
+	 */
 }
 
 void
@@ -157,7 +200,36 @@ psset_update_end(psset_t *psset, hpdata_t *ps) {
 	hpdata_assert_consistent(ps);
 	assert(hpdata_in_psset_get(ps));
 	hpdata_updating_set(ps, false);
-	psset_do_alloc_tracking_insert(psset, ps);
+	psset_stats_insert(psset, ps);
+
+	/*
+	 * The update begin should have removed ps from whatever alloc container
+	 * it was in.
+	 */
+	assert(!hpdata_in_psset_alloc_container_get(ps));
+	if (hpdata_alloc_allowed_get(ps)) {
+		psset_alloc_container_insert(psset, ps);
+	}
+
+	if (hpdata_purge_allowed_get(ps)
+	    && !hpdata_in_psset_purge_container_get(ps)) {
+		hpdata_in_psset_purge_container_set(ps, true);
+		hpdata_purge_list_append(&psset->to_purge, ps);
+	} else if (!hpdata_purge_allowed_get(ps)
+	    && hpdata_in_psset_purge_container_get(ps)) {
+		hpdata_in_psset_purge_container_set(ps, false);
+		hpdata_purge_list_remove(&psset->to_purge, ps);
+	}
+
+	if (hpdata_hugify_allowed_get(ps)
+	    && !hpdata_in_psset_hugify_container_get(ps)) {
+		hpdata_in_psset_hugify_container_set(ps, true);
+		hpdata_hugify_list_append(&psset->to_hugify, ps);
+	} else if (!hpdata_hugify_allowed_get(ps)
+	    && hpdata_in_psset_hugify_container_get(ps)) {
+		hpdata_in_psset_hugify_container_set(ps, false);
+		hpdata_hugify_list_remove(&psset->to_hugify, ps);
+	}
 }
 
 hpdata_t *
@@ -169,7 +241,7 @@ psset_pick_alloc(psset_t *psset, size_t size) {
 	pszind_t pind = (pszind_t)bitmap_ffu(psset->bitmap, &psset_bitmap_info,
 	    (size_t)min_pind);
 	if (pind == PSSET_NPSIZES) {
-		return hpdata_empty_list_first(&psset->empty_slabs);
+		return hpdata_empty_list_first(&psset->empty);
 	}
 	hpdata_t *ps = hpdata_age_heap_first(&psset->pageslabs[pind]);
 	if (ps == NULL) {
@@ -181,16 +253,48 @@ psset_pick_alloc(psset_t *psset, size_t size) {
 	return ps;
 }
 
+hpdata_t *
+psset_pick_purge(psset_t *psset) {
+	return hpdata_purge_list_first(&psset->to_purge);
+}
+
+hpdata_t *
+psset_pick_hugify(psset_t *psset) {
+	return hpdata_hugify_list_first(&psset->to_hugify);
+}
+
 void
 psset_insert(psset_t *psset, hpdata_t *ps) {
-	/* We only support inserting empty pageslabs, for now. */
-	assert(hpdata_empty(ps));
 	hpdata_in_psset_set(ps, true);
-	psset_do_alloc_tracking_insert(psset, ps);
+
+	psset_stats_insert(psset, ps);
+	if (hpdata_alloc_allowed_get(ps)) {
+		psset_alloc_container_insert(psset, ps);
+	}
+	if (hpdata_purge_allowed_get(ps)) {
+		hpdata_in_psset_purge_container_set(ps, true);
+		hpdata_purge_list_append(&psset->to_purge, ps);
+	}
+	if (hpdata_hugify_allowed_get(ps)) {
+		hpdata_in_psset_hugify_container_set(ps, true);
+		hpdata_hugify_list_append(&psset->to_hugify, ps);
+	}
 }
 
 void
 psset_remove(psset_t *psset, hpdata_t *ps) {
 	hpdata_in_psset_set(ps, false);
-	psset_do_alloc_tracking_remove(psset, ps);
+
+	psset_stats_remove(psset, ps);
+	if (hpdata_in_psset_alloc_container_get(ps)) {
+		psset_alloc_container_remove(psset, ps);
+	}
+	if (hpdata_in_psset_purge_container_get(ps)) {
+		hpdata_in_psset_purge_container_set(ps, false);
+		hpdata_purge_list_remove(&psset->to_purge, ps);
+	}
+	if (hpdata_in_psset_purge_container_get(ps)) {
+		hpdata_in_psset_purge_container_set(ps, false);
+		hpdata_purge_list_remove(&psset->to_purge, ps);
+	}
 }