Move VM reserve/release/purge/commit/decommit/guards/trim to OS layer

Migrates some VM functions namely, reserve, release, purge, commit,
decommit, guards (mark and unmark), and trim to OS layer.

include/jemalloc/internal/os.h

@@ -46,4 +46,7 @@
 /* CPU */
 #include "jemalloc/internal/os/cpu.h"
 
+/* VM */
+#include "jemalloc/internal/os/vm.h"
+
 #endif /* JEMALLOC_INTERNAL_OS_H */

include/jemalloc/internal/os/posix/vm.h

@@ -0,0 +1,290 @@
+#ifndef JEMALLOC_INTERNAL_OS_POSIX_VM_H
+#define JEMALLOC_INTERNAL_OS_POSIX_VM_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+#include "jemalloc/internal/bit_util.h"
+
+#include <sys/mman.h>
+#if defined(JEMALLOC_HAVE_PRCTL) && defined(JEMALLOC_PAGEID)
+#  include <sys/prctl.h>
+#  ifndef PR_SET_VMA
+#    define PR_SET_VMA 0x53564d41
+#    define PR_SET_VMA_ANON_NAME 0
+#  endif
+#endif
+#ifdef __NetBSD__
+#  include <sys/bitops.h> /* ilog2 */
+#endif
+
+#ifdef JEMALLOC_HAVE_VM_MAKE_TAG
+#  define PAGES_FD_TAG VM_MAKE_TAG(254U)
+#else
+#  define PAGES_FD_TAG -1
+#endif
+#define PAGES_PROT_COMMIT   (PROT_READ | PROT_WRITE)
+#define PAGES_PROT_DECOMMIT (PROT_NONE)
+
+/* mmap flags assembled by pages_boot(); MAP_PRIVATE | MAP_ANON (+ MAP_NORESERVE
+ * when the kernel overcommits). */
+extern int mmap_flags;
+
+#ifdef JEMALLOC_PAGEID
+JEMALLOC_ALWAYS_INLINE int
+os_page_id(void *addr, size_t size, const char *name) {
+#  ifdef JEMALLOC_HAVE_PRCTL
+	/*
+	 * While parsing `/proc/<pid>/maps` file, the block could appear as
+	 * 7f4836000000-7f4836800000 rw-p 00000000 00:00 0 [anon:jemalloc_pg_overcommit]`
+	 */
+	int n;
+	assert(addr != NULL);
+	n = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)addr, size,
+	    (uintptr_t)name);
+	assert(n == 0 || (n == -1 && get_errno() == EINVAL));
+	return n;
+#  else
+	return 0;
+#  endif
+}
+#endif
+
+JEMALLOC_ALWAYS_INLINE void *
+os_vm_reserve(void *hint, size_t size, size_t alignment, bool *commit) {
+	assert(ALIGNMENT_ADDR2BASE(hint, os_page) == hint);
+	assert(ALIGNMENT_CEILING(size, os_page) == size);
+	assert(size != 0);
+
+	if (os_overcommits) {
+		*commit = true;
+	}
+
+	void *ret;
+	/*
+	 * We don't use MAP_FIXED here, because it can cause the *replacement*
+	 * of existing mappings, and we only want to create new mappings.
+	 */
+	{
+		int flags = mmap_flags;
+#ifdef __NetBSD__
+		/*
+		 * On NetBSD PAGE for a platform is defined to the
+		 * maximum page size of all machine architectures
+		 * for that platform, so that we can use the same
+		 * binaries across all machine architectures.
+		 */
+		if (alignment > os_page || PAGE > os_page) {
+			unsigned int a = ilog2(MAX(alignment, PAGE));
+			flags |= MAP_ALIGNED(a);
+		}
+#endif
+		int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
+
+		ret = mmap(hint, size, prot, flags, PAGES_FD_TAG, 0);
+	}
+	assert(ret != NULL);
+
+	if (ret == MAP_FAILED) {
+		ret = NULL;
+	} else if (hint != NULL && ret != hint) {
+		/*
+		 * We succeeded in mapping memory, but not in the right place.
+		 */
+		os_vm_release(ret, size);
+		ret = NULL;
+	}
+	assert(ret == NULL || (hint == NULL && ret != hint)
+	    || (hint != NULL && ret == hint));
+#ifdef JEMALLOC_PAGEID
+	if (ret != NULL) {
+		os_page_id(ret, size,
+		    os_overcommits ? "jemalloc_pg_overcommit" : "jemalloc_pg");
+	}
+#endif
+	return ret;
+}
+
+#if defined(__FreeBSD__) && defined(MAP_EXCL)
+/*
+ * FreeBSD has mechanisms both to mmap at a specific address without
+ * touching existing mappings (MAP_EXCL), and to mmap with a specific
+ * alignment.
+ */
+#  define OS_VM_HAS_FIXED_ALIGNED_RESERVE
+
+JEMALLOC_ALWAYS_INLINE void *
+os_vm_reserve_aligned_fixed(void *addr, size_t size, size_t alignment,
+    bool *commit) {
+	if (os_overcommits) {
+		*commit = true;
+	}
+
+	int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
+	int flags = mmap_flags;
+
+	if (addr != NULL) {
+		flags |= MAP_FIXED | MAP_EXCL;
+	} else {
+		unsigned alignment_bits = ffs_zu(alignment);
+		assert(alignment_bits > 0);
+		flags |= MAP_ALIGNED(alignment_bits);
+	}
+
+	void *ret = mmap(addr, size, prot, flags, -1, 0);
+	if (ret == MAP_FAILED) {
+		ret = NULL;
+	}
+
+	return ret;
+}
+#endif
+
+JEMALLOC_ALWAYS_INLINE void
+os_vm_release(void *addr, size_t size) {
+	assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
+	assert(ALIGNMENT_CEILING(size, os_page) == size);
+
+	if (munmap(addr, size) == -1) {
+		char buf[BUFERROR_BUF];
+
+		buferror(get_errno(), buf, sizeof(buf));
+		malloc_printf("<jemalloc>: Error in munmap(): %s\n", buf);
+		if (opt_abort) {
+			abort();
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+os_vm_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
+    bool *commit) {
+	assert(alloc_size >= leadsize + size);
+	void  *ret = (void *)((byte_t *)addr + leadsize);
+	size_t trailsize = alloc_size - leadsize - size;
+
+	(void)commit;
+	if (leadsize != 0) {
+		os_vm_release(addr, leadsize);
+	}
+	if (trailsize != 0) {
+		os_vm_release((void *)((byte_t *)ret + size), trailsize);
+	}
+	return ret;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_commit_impl(void *addr, size_t size, bool commit) {
+	assert(PAGE_ADDR2BASE(addr) == addr);
+	assert(PAGE_CEILING(size) == size);
+
+	int   prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
+	void *result = mmap(
+	    addr, size, prot, mmap_flags | MAP_FIXED, PAGES_FD_TAG, 0);
+	if (result == MAP_FAILED) {
+		return true;
+	}
+	if (result != addr) {
+		/*
+		 * We succeeded in mapping memory, but not in the right
+		 * place.
+		 */
+		os_vm_release(result, size);
+		return true;
+	}
+	return false;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_commit(void *addr, size_t size) {
+	return os_vm_commit_impl(addr, size, true);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_decommit(void *addr, size_t size) {
+	return os_vm_commit_impl(addr, size, false);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_vm_mark_guards(void *head, void *tail) {
+	assert(head != NULL || tail != NULL);
+	assert(
+	    head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
+#ifdef JEMALLOC_HAVE_MPROTECT
+	if (head != NULL) {
+		mprotect(head, PAGE, PROT_NONE);
+	}
+	if (tail != NULL) {
+		mprotect(tail, PAGE, PROT_NONE);
+	}
+#else
+	/* Decommit sets the region to PROT_NONE. */
+	if (head != NULL) {
+		os_vm_decommit(head, PAGE);
+	}
+	if (tail != NULL) {
+		os_vm_decommit(tail, PAGE);
+	}
+#endif
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_vm_unmark_guards(void *head, void *tail) {
+	assert(head != NULL || tail != NULL);
+	assert(
+	    head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
+#ifdef JEMALLOC_HAVE_MPROTECT
+	bool   head_and_tail = (head != NULL) && (tail != NULL);
+	size_t range = head_and_tail ? (uintptr_t)tail - (uintptr_t)head + PAGE
+	                             : SIZE_T_MAX;
+	/*
+	 * The amount of work that the kernel does in mprotect depends on the
+	 * range argument.  SC_LARGE_MINCLASS is an arbitrary threshold chosen
+	 * to prevent kernel from doing too much work that would outweigh the
+	 * savings of performing one less system call.
+	 */
+	bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS;
+	if (ranged_mprotect) {
+		mprotect(head, range, PROT_READ | PROT_WRITE);
+	} else {
+		if (head != NULL) {
+			mprotect(head, PAGE, PROT_READ | PROT_WRITE);
+		}
+		if (tail != NULL) {
+			mprotect(tail, PAGE, PROT_READ | PROT_WRITE);
+		}
+	}
+#else
+	if (head != NULL) {
+		os_vm_commit(head, PAGE);
+	}
+	if (tail != NULL) {
+		os_vm_commit(tail, PAGE);
+	}
+#endif
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_purge_lazy(void *addr, size_t size) {
+#if defined(JEMALLOC_PURGE_MADVISE_FREE)
+	return (madvise(addr, size,
+#  ifdef MADV_FREE
+	            MADV_FREE
+#  else
+	            JEMALLOC_MADV_FREE
+#  endif
+	            )
+	    != 0);
+#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)                                \
+    && !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
+	return (madvise(addr, size, MADV_DONTNEED) != 0);
+#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED)                          \
+    && !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
+	return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
+#else
+	(void)addr;
+	(void)size;
+	not_reached();
+#endif
+}
+
+#endif /* JEMALLOC_INTERNAL_OS_POSIX_VM_H */

include/jemalloc/internal/os/vm.h

@@ -0,0 +1,44 @@
+#ifndef JEMALLOC_INTERNAL_OS_VM_H
+#define JEMALLOC_INTERNAL_OS_VM_H
+
+#include "jemalloc/internal/jemalloc_internal_externs.h"
+#include "jemalloc/internal/jemalloc_preamble.h"
+#include "jemalloc/internal/malloc_io.h"
+#include "jemalloc/internal/os/detect.h"
+#include "jemalloc/internal/pages.h"
+#include "jemalloc/internal/sc.h"
+
+/*
+ * VM interface: raw page-level reserve/commit/decommit/guard/trim primitives
+ * used by src/pages.c. Platform-specific fast paths (e.g. the FreeBSD
+ * MAP_EXCL reservation shortcut) stay in src/pages.c, since they are
+ * portable decisions layered on top of these primitives, not primitives
+ * themselves.
+ * Default: posix/.  Override: Windows (VirtualAlloc/VirtualFree).
+ *
+ * State: `os_overcommits`, set by pages_boot(), read directly by the
+ * backends below.
+ */
+extern bool os_overcommits;
+
+/* Functions required for implementation in each backend. */
+JEMALLOC_ALWAYS_INLINE void *os_vm_reserve(
+    void *hint, size_t size, size_t alignment, bool *commit);
+JEMALLOC_ALWAYS_INLINE void os_vm_release(void *addr, size_t size);
+JEMALLOC_ALWAYS_INLINE void *os_vm_trim(void *addr, size_t alloc_size,
+    size_t leadsize, size_t size, bool *commit);
+JEMALLOC_ALWAYS_INLINE bool os_vm_commit(void *addr, size_t size);
+JEMALLOC_ALWAYS_INLINE bool os_vm_decommit(void *addr, size_t size);
+JEMALLOC_ALWAYS_INLINE void os_vm_mark_guards(void *head, void *tail);
+JEMALLOC_ALWAYS_INLINE void os_vm_unmark_guards(void *head, void *tail);
+JEMALLOC_ALWAYS_INLINE bool os_vm_purge_lazy(void *addr, size_t size);
+
+#if defined(_WIN32)
+#  include "jemalloc/internal/os/windows/vm.h"
+#elif defined(JEMALLOC_OS_POSIX)
+#  include "jemalloc/internal/os/posix/vm.h"
+#else
+#  error "OS layer: no vm backend for this platform; add os/<os>/vm.h"
+#endif
+
+#endif /* JEMALLOC_INTERNAL_OS_VM_H */

include/jemalloc/internal/os/windows/vm.h

@@ -0,0 +1,115 @@
+#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_VM_H
+#define JEMALLOC_INTERNAL_OS_WINDOWS_VM_H
+
+#include "jemalloc/internal/jemalloc_preamble.h"
+
+JEMALLOC_ALWAYS_INLINE void *
+os_vm_reserve(void *hint, size_t size, size_t alignment, bool *commit) {
+	assert(ALIGNMENT_ADDR2BASE(hint, os_page) == hint);
+	assert(ALIGNMENT_CEILING(size, os_page) == size);
+	assert(size != 0);
+	(void)alignment;
+
+	if (os_overcommits) {
+		*commit = true;
+	}
+
+	void *ret;
+	/*
+	 * If VirtualAlloc can't allocate at the given address when one is
+	 * given, it fails and returns NULL.
+	 */
+	ret = VirtualAlloc(hint, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
+	    PAGE_READWRITE);
+	assert(ret == NULL || (hint == NULL && ret != hint)
+	    || (hint != NULL && ret == hint));
+	return ret;
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_vm_release(void *addr, size_t size) {
+	assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
+	assert(ALIGNMENT_CEILING(size, os_page) == size);
+
+	if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
+		char buf[BUFERROR_BUF];
+
+		buferror(get_errno(), buf, sizeof(buf));
+		malloc_printf("<jemalloc>: Error in VirtualFree(): %s\n", buf);
+		if (opt_abort) {
+			abort();
+		}
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void *
+os_vm_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
+    bool *commit) {
+	assert(alloc_size >= leadsize + size);
+	void *ret = (void *)((byte_t *)addr + leadsize);
+
+	os_vm_release(addr, alloc_size);
+	void *new_addr = os_vm_reserve(ret, size, PAGE, commit);
+	if (new_addr == ret) {
+		return ret;
+	}
+	if (new_addr != NULL) {
+		os_vm_release(new_addr, size);
+	}
+	return NULL;
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_commit_impl(void *addr, size_t size, bool commit) {
+	assert(PAGE_ADDR2BASE(addr) == addr);
+	assert(PAGE_CEILING(size) == size);
+
+	return (commit
+	        ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE))
+	        : (!VirtualFree(addr, size, MEM_DECOMMIT)));
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_commit(void *addr, size_t size) {
+	return os_vm_commit_impl(addr, size, true);
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_decommit(void *addr, size_t size) {
+	return os_vm_commit_impl(addr, size, false);
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_vm_mark_guards(void *head, void *tail) {
+	assert(head != NULL || tail != NULL);
+	assert(
+	    head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
+	/* Decommit sets the region to MEM_DECOMMIT. */
+	if (head != NULL) {
+		os_vm_decommit(head, PAGE);
+	}
+	if (tail != NULL) {
+		os_vm_decommit(tail, PAGE);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE void
+os_vm_unmark_guards(void *head, void *tail) {
+	assert(head != NULL || tail != NULL);
+	assert(
+	    head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
+	if (head != NULL) {
+		os_vm_commit(head, PAGE);
+	}
+	if (tail != NULL) {
+		os_vm_commit(tail, PAGE);
+	}
+}
+
+JEMALLOC_ALWAYS_INLINE bool
+os_vm_purge_lazy(void *addr, size_t size) {
+	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
+	return false;
+}
+
+#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_VM_H */

src/pages.c

@@ -14,21 +14,6 @@
 #		include <vm/vm_param.h>
 #	endif
 #endif
-#ifdef __NetBSD__
-#	include <sys/bitops.h> /* ilog2 */
-#endif
-#ifdef JEMALLOC_HAVE_VM_MAKE_TAG
-#	define PAGES_FD_TAG VM_MAKE_TAG(254U)
-#else
-#	define PAGES_FD_TAG -1
-#endif
-#if defined(JEMALLOC_HAVE_PRCTL) && defined(JEMALLOC_PAGEID)
-#	include <sys/prctl.h>
-#	ifndef PR_SET_VMA
-#		define PR_SET_VMA 0x53564d41
-#		define PR_SET_VMA_ANON_NAME 0
-#	endif
-#endif
 
 /******************************************************************************/
 /* Data. */
@@ -36,12 +21,10 @@
 /* Actual operating system page size, detected during bootstrap, <= PAGE. */
 size_t os_page;
 
-#ifndef _WIN32
+/* Set here. Consumed by os_vm_reserve/os_vm_commit in os/posix/vm.h. */
-#	define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
+int mmap_flags;
-#	define PAGES_PROT_DECOMMIT (PROT_NONE)
+/* Set here, consumed directly by the os/vm.h backends. */
-static int mmap_flags;
+bool os_overcommits;
-#endif
-static bool os_overcommits;
 
 const char *const thp_mode_names[] = {
     "default", "always", "never", "not supported"};
@@ -106,156 +89,12 @@ madvise_MADV_DONTNEED_zeroes_pages(void) {
 }
 #endif
 
-#ifdef JEMALLOC_PAGEID
-static int
-os_page_id(void *addr, size_t size, const char *name) {
-#	ifdef JEMALLOC_HAVE_PRCTL
-	/*
-	 * While parsing `/proc/<pid>/maps` file, the block could appear as
-	 * 7f4836000000-7f4836800000 rw-p 00000000 00:00 0 [anon:jemalloc_pg_overcommit]`
-	 */
-	int n;
-	assert(addr != NULL);
-	n = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)addr, size,
-	    (uintptr_t)name);
-	assert(n == 0 || (n == -1 && get_errno() == EINVAL));
-	return n;
-#	else
-	return 0;
-#	endif
-}
-#endif
-
-/******************************************************************************/
-/*
- * Function prototypes for static functions that are referenced prior to
- * definition.
- */
-
-static void os_pages_unmap(void *addr, size_t size);
-
 /******************************************************************************/
 
 static void *
-os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
+pages_trim(
-	assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
-	assert(ALIGNMENT_CEILING(size, os_page) == size);
-	assert(size != 0);
-
-	if (os_overcommits) {
-		*commit = true;
-	}
-
-	void *ret;
-#ifdef _WIN32
-	/*
-	 * If VirtualAlloc can't allocate at the given address when one is
-	 * given, it fails and returns NULL.
-	 */
-	ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
-	    PAGE_READWRITE);
-#else
-	/*
-	 * We don't use MAP_FIXED here, because it can cause the *replacement*
-	 * of existing mappings, and we only want to create new mappings.
-	 */
-	{
-		int flags = mmap_flags;
-#	ifdef __NetBSD__
-		/*
-		 * On NetBSD PAGE for a platform is defined to the
-		 * maximum page size of all machine architectures
-		 * for that platform, so that we can use the same
-		 * binaries across all machine architectures.
-		 */
-		if (alignment > os_page || PAGE > os_page) {
-			unsigned int a = ilog2(MAX(alignment, PAGE));
-			flags |= MAP_ALIGNED(a);
-		}
-#	endif
-		int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
-
-		ret = mmap(addr, size, prot, flags, PAGES_FD_TAG, 0);
-	}
-	assert(ret != NULL);
-
-	if (ret == MAP_FAILED) {
-		ret = NULL;
-	} else if (addr != NULL && ret != addr) {
-		/*
-		 * We succeeded in mapping memory, but not in the right place.
-		 */
-		os_pages_unmap(ret, size);
-		ret = NULL;
-	}
-#endif
-	assert(ret == NULL || (addr == NULL && ret != addr)
-	    || (addr != NULL && ret == addr));
-#ifdef JEMALLOC_PAGEID
-	if (ret != NULL) {
-		os_page_id(ret, size,
-		    os_overcommits ? "jemalloc_pg_overcommit" : "jemalloc_pg");
-	}
-#endif
-	return ret;
-}
-
-static void *
-os_pages_trim(
     void *addr, size_t alloc_size, size_t leadsize, size_t size, bool *commit) {
-	void *ret = (void *)((byte_t *)addr + leadsize);
+	return os_vm_trim(addr, alloc_size, leadsize, size, commit);
-
-	assert(alloc_size >= leadsize + size);
-#ifdef _WIN32
-	os_pages_unmap(addr, alloc_size);
-	void *new_addr = os_pages_map(ret, size, PAGE, commit);
-	if (new_addr == ret) {
-		return ret;
-	}
-	if (new_addr != NULL) {
-		os_pages_unmap(new_addr, size);
-	}
-	return NULL;
-#else
-	size_t trailsize = alloc_size - leadsize - size;
-
-	if (leadsize != 0) {
-		os_pages_unmap(addr, leadsize);
-	}
-	if (trailsize != 0) {
-		os_pages_unmap((void *)((byte_t *)ret + size), trailsize);
-	}
-	return ret;
-#endif
-}
-
-static void
-os_pages_unmap(void *addr, size_t size) {
-	assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
-	assert(ALIGNMENT_CEILING(size, os_page) == size);
-
-#ifdef _WIN32
-	if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
-#else
-	if (munmap(addr, size) == -1)
-#endif
-	{
-		char buf[BUFERROR_BUF];
-
-		buferror(get_errno(), buf, sizeof(buf));
-		malloc_printf(
-		    "<jemalloc>: Error in "
-#ifdef _WIN32
-		    "VirtualFree"
-#else
-		    "munmap"
-#endif
-		    "(): %s\n",
-		    buf);
-		if (opt_abort) {
-			abort();
-		}
-	}
 }
 
 static void *
@@ -268,13 +107,13 @@ pages_map_slow(size_t size, size_t alignment, bool *commit) {
 
 	void *ret;
 	do {
-		void *pages = os_pages_map(NULL, alloc_size, alignment, commit);
+		void *pages = os_vm_reserve(NULL, alloc_size, alignment, commit);
 		if (pages == NULL) {
 			return NULL;
 		}
 		size_t leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment)
 		    - (uintptr_t)pages;
-		ret = os_pages_trim(pages, alloc_size, leadsize, size, commit);
+		ret = pages_trim(pages, alloc_size, leadsize, size, commit);
 	} while (ret == NULL);
 
 	assert(ret != NULL);
@@ -287,34 +126,8 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
 	assert(alignment >= PAGE);
 	assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr);
 
-#if defined(__FreeBSD__) && defined(MAP_EXCL)
+#ifdef OS_VM_HAS_FIXED_ALIGNED_RESERVE
-	/*
+	return os_vm_reserve_aligned_fixed(addr, size, alignment, commit);
-	 * FreeBSD has mechanisms both to mmap at specific address without
-	 * touching existing mappings, and to mmap with specific alignment.
-	 */
-	{
-		if (os_overcommits) {
-			*commit = true;
-		}
-
-		int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
-		int flags = mmap_flags;
-
-		if (addr != NULL) {
-			flags |= MAP_FIXED | MAP_EXCL;
-		} else {
-			unsigned alignment_bits = ffs_zu(alignment);
-			assert(alignment_bits > 0);
-			flags |= MAP_ALIGNED(alignment_bits);
-		}
-
-		void *ret = mmap(addr, size, prot, flags, -1, 0);
-		if (ret == MAP_FAILED) {
-			ret = NULL;
-		}
-
-		return ret;
-	}
 #endif
 	/*
 	 * Ideally, there would be a way to specify alignment to mmap() (like
@@ -322,7 +135,7 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
 	 * hard to efficiently create aligned mappings.  The reliable, but
 	 * slow method is to create a mapping that is over-sized, then trim the
 	 * excess.  However, that always results in one or two calls to
-	 * os_pages_unmap(), and it can leave holes in the process's virtual
+	 * os_vm_release(), and it can leave holes in the process's virtual
 	 * memory map if memory grows downward.
 	 *
 	 * Optimistically try mapping precisely the right amount before falling
@@ -330,13 +143,13 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
 	 * approach works most of the time.
 	 */
 
-	void *ret = os_pages_map(addr, size, os_page, commit);
+	void *ret = os_vm_reserve(addr, size, os_page, commit);
 	if (ret == NULL || ret == addr) {
 		return ret;
 	}
 	assert(addr == NULL);
 	if (ALIGNMENT_ADDR2OFFSET(ret, alignment) != 0) {
-		os_pages_unmap(ret, size);
+		os_vm_release(ret, size);
 		return pages_map_slow(size, alignment, commit);
 	}
 
@@ -349,37 +162,7 @@ pages_unmap(void *addr, size_t size) {
 	assert(PAGE_ADDR2BASE(addr) == addr);
 	assert(PAGE_CEILING(size) == size);
 
-	os_pages_unmap(addr, size);
+	os_vm_release(addr, size);
-}
-
-static bool
-os_pages_commit(void *addr, size_t size, bool commit) {
-	assert(PAGE_ADDR2BASE(addr) == addr);
-	assert(PAGE_CEILING(size) == size);
-
-#ifdef _WIN32
-	return (commit
-	        ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE))
-	        : (!VirtualFree(addr, size, MEM_DECOMMIT)));
-#else
-	{
-		int   prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
-		void *result = mmap(
-		    addr, size, prot, mmap_flags | MAP_FIXED, PAGES_FD_TAG, 0);
-		if (result == MAP_FAILED) {
-			return true;
-		}
-		if (result != addr) {
-			/*
-			 * We succeeded in mapping memory, but not in the right
-			 * place.
-			 */
-			os_pages_unmap(result, size);
-			return true;
-		}
-		return false;
-	}
-#endif
 }
 
 static bool
@@ -388,7 +171,7 @@ pages_commit_impl(void *addr, size_t size, bool commit) {
 		return true;
 	}
 
-	return os_pages_commit(addr, size, commit);
+	return commit ? os_vm_commit(addr, size) : os_vm_decommit(addr, size);
 }
 
 bool
@@ -403,61 +186,12 @@ pages_decommit(void *addr, size_t size) {
 
 void
 pages_mark_guards(void *head, void *tail) {
-	assert(head != NULL || tail != NULL);
+	os_vm_mark_guards(head, tail);
-	assert(
-	    head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
-#ifdef JEMALLOC_HAVE_MPROTECT
-	if (head != NULL) {
-		mprotect(head, PAGE, PROT_NONE);
-	}
-	if (tail != NULL) {
-		mprotect(tail, PAGE, PROT_NONE);
-	}
-#else
-	/* Decommit sets to PROT_NONE / MEM_DECOMMIT. */
-	if (head != NULL) {
-		os_pages_commit(head, PAGE, false);
-	}
-	if (tail != NULL) {
-		os_pages_commit(tail, PAGE, false);
-	}
-#endif
 }
 
 void
 pages_unmark_guards(void *head, void *tail) {
-	assert(head != NULL || tail != NULL);
+	os_vm_unmark_guards(head, tail);
-	assert(
-	    head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
-#ifdef JEMALLOC_HAVE_MPROTECT
-	bool   head_and_tail = (head != NULL) && (tail != NULL);
-	size_t range = head_and_tail ? (uintptr_t)tail - (uintptr_t)head + PAGE
-	                             : SIZE_T_MAX;
-	/*
-	 * The amount of work that the kernel does in mprotect depends on the
-	 * range argument.  SC_LARGE_MINCLASS is an arbitrary threshold chosen
-	 * to prevent kernel from doing too much work that would outweigh the
-	 * savings of performing one less system call.
-	 */
-	bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS;
-	if (ranged_mprotect) {
-		mprotect(head, range, PROT_READ | PROT_WRITE);
-	} else {
-		if (head != NULL) {
-			mprotect(head, PAGE, PROT_READ | PROT_WRITE);
-		}
-		if (tail != NULL) {
-			mprotect(tail, PAGE, PROT_READ | PROT_WRITE);
-		}
-	}
-#else
-	if (head != NULL) {
-		os_pages_commit(head, PAGE, true);
-	}
-	if (tail != NULL) {
-		os_pages_commit(tail, PAGE, true);
-	}
-#endif
 }
 
 bool
@@ -476,27 +210,7 @@ pages_purge_lazy(void *addr, size_t size) {
 		return true;
 	}
 
-#ifdef _WIN32
+	return os_vm_purge_lazy(addr, size);
-	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
-	return false;
-#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
-	return (madvise(addr, size,
-#	ifdef MADV_FREE
-	            MADV_FREE
-#	else
-	            JEMALLOC_MADV_FREE
-#	endif
-	            )
-	    != 0);
-#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED)                                 \
-    && !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
-	return (madvise(addr, size, MADV_DONTNEED) != 0);
-#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED)                           \
-    && !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
-	return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
-#else
-	not_reached();
-#endif
 }
 
 bool
@@ -932,7 +646,7 @@ pages_boot(void) {
 	/* Detect lazy purge runtime support. */
 	if (pages_can_purge_lazy) {
 		bool  committed = false;
-		void *madv_free_page = os_pages_map(
+		void *madv_free_page = os_vm_reserve(
 		    NULL, PAGE, PAGE, &committed);
 		if (madv_free_page == NULL) {
 			return true;
@@ -941,7 +655,7 @@ pages_boot(void) {
 		if (pages_purge_lazy(madv_free_page, PAGE)) {
 			pages_can_purge_lazy_runtime = false;
 		}
-		os_pages_unmap(madv_free_page, PAGE);
+		os_vm_release(madv_free_page, PAGE);
 	}
 #endif
 	if (init_process_madvise()) {