From c8e2e0134cd13350e33f77515c00e5eb170dc683 Mon Sep 17 00:00:00 2001 From: guangli-dai Date: Mon, 27 Jul 2026 20:12:33 -0700 Subject: [PATCH] Move VM reserve/release/purge/commit/decommit/guards/trim to OS layer Migrates some VM functions namely, reserve, release, purge, commit, decommit, guards (mark and unmark), and trim to OS layer. --- include/jemalloc/internal/os.h | 3 + include/jemalloc/internal/os/posix/vm.h | 290 +++++++++++++++++++ include/jemalloc/internal/os/vm.h | 44 +++ include/jemalloc/internal/os/windows/vm.h | 115 ++++++++ src/pages.c | 326 ++-------------------- 5 files changed, 472 insertions(+), 306 deletions(-) create mode 100644 include/jemalloc/internal/os/posix/vm.h create mode 100644 include/jemalloc/internal/os/vm.h create mode 100644 include/jemalloc/internal/os/windows/vm.h diff --git a/include/jemalloc/internal/os.h b/include/jemalloc/internal/os.h index 4354fb42..d232eb71 100644 --- a/include/jemalloc/internal/os.h +++ b/include/jemalloc/internal/os.h @@ -46,4 +46,7 @@ /* CPU */ #include "jemalloc/internal/os/cpu.h" +/* VM */ +#include "jemalloc/internal/os/vm.h" + #endif /* JEMALLOC_INTERNAL_OS_H */ diff --git a/include/jemalloc/internal/os/posix/vm.h b/include/jemalloc/internal/os/posix/vm.h new file mode 100644 index 00000000..81c40440 --- /dev/null +++ b/include/jemalloc/internal/os/posix/vm.h @@ -0,0 +1,290 @@ +#ifndef JEMALLOC_INTERNAL_OS_POSIX_VM_H +#define JEMALLOC_INTERNAL_OS_POSIX_VM_H + +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/bit_util.h" + +#include +#if defined(JEMALLOC_HAVE_PRCTL) && defined(JEMALLOC_PAGEID) +# include +# ifndef PR_SET_VMA +# define PR_SET_VMA 0x53564d41 +# define PR_SET_VMA_ANON_NAME 0 +# endif +#endif +#ifdef __NetBSD__ +# include /* ilog2 */ +#endif + +#ifdef JEMALLOC_HAVE_VM_MAKE_TAG +# define PAGES_FD_TAG VM_MAKE_TAG(254U) +#else +# define PAGES_FD_TAG -1 +#endif +#define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE) +#define PAGES_PROT_DECOMMIT (PROT_NONE) + +/* mmap flags assembled by pages_boot(); MAP_PRIVATE | MAP_ANON (+ MAP_NORESERVE + * when the kernel overcommits). */ +extern int mmap_flags; + +#ifdef JEMALLOC_PAGEID +JEMALLOC_ALWAYS_INLINE int +os_page_id(void *addr, size_t size, const char *name) { +# ifdef JEMALLOC_HAVE_PRCTL + /* + * While parsing `/proc//maps` file, the block could appear as + * 7f4836000000-7f4836800000 rw-p 00000000 00:00 0 [anon:jemalloc_pg_overcommit]` + */ + int n; + assert(addr != NULL); + n = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)addr, size, + (uintptr_t)name); + assert(n == 0 || (n == -1 && get_errno() == EINVAL)); + return n; +# else + return 0; +# endif +} +#endif + +JEMALLOC_ALWAYS_INLINE void * +os_vm_reserve(void *hint, size_t size, size_t alignment, bool *commit) { + assert(ALIGNMENT_ADDR2BASE(hint, os_page) == hint); + assert(ALIGNMENT_CEILING(size, os_page) == size); + assert(size != 0); + + if (os_overcommits) { + *commit = true; + } + + void *ret; + /* + * We don't use MAP_FIXED here, because it can cause the *replacement* + * of existing mappings, and we only want to create new mappings. + */ + { + int flags = mmap_flags; +#ifdef __NetBSD__ + /* + * On NetBSD PAGE for a platform is defined to the + * maximum page size of all machine architectures + * for that platform, so that we can use the same + * binaries across all machine architectures. + */ + if (alignment > os_page || PAGE > os_page) { + unsigned int a = ilog2(MAX(alignment, PAGE)); + flags |= MAP_ALIGNED(a); + } +#endif + int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; + + ret = mmap(hint, size, prot, flags, PAGES_FD_TAG, 0); + } + assert(ret != NULL); + + if (ret == MAP_FAILED) { + ret = NULL; + } else if (hint != NULL && ret != hint) { + /* + * We succeeded in mapping memory, but not in the right place. + */ + os_vm_release(ret, size); + ret = NULL; + } + assert(ret == NULL || (hint == NULL && ret != hint) + || (hint != NULL && ret == hint)); +#ifdef JEMALLOC_PAGEID + if (ret != NULL) { + os_page_id(ret, size, + os_overcommits ? "jemalloc_pg_overcommit" : "jemalloc_pg"); + } +#endif + return ret; +} + +#if defined(__FreeBSD__) && defined(MAP_EXCL) +/* + * FreeBSD has mechanisms both to mmap at a specific address without + * touching existing mappings (MAP_EXCL), and to mmap with a specific + * alignment. + */ +# define OS_VM_HAS_FIXED_ALIGNED_RESERVE + +JEMALLOC_ALWAYS_INLINE void * +os_vm_reserve_aligned_fixed(void *addr, size_t size, size_t alignment, + bool *commit) { + if (os_overcommits) { + *commit = true; + } + + int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; + int flags = mmap_flags; + + if (addr != NULL) { + flags |= MAP_FIXED | MAP_EXCL; + } else { + unsigned alignment_bits = ffs_zu(alignment); + assert(alignment_bits > 0); + flags |= MAP_ALIGNED(alignment_bits); + } + + void *ret = mmap(addr, size, prot, flags, -1, 0); + if (ret == MAP_FAILED) { + ret = NULL; + } + + return ret; +} +#endif + +JEMALLOC_ALWAYS_INLINE void +os_vm_release(void *addr, size_t size) { + assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); + assert(ALIGNMENT_CEILING(size, os_page) == size); + + if (munmap(addr, size) == -1) { + char buf[BUFERROR_BUF]; + + buferror(get_errno(), buf, sizeof(buf)); + malloc_printf(": Error in munmap(): %s\n", buf); + if (opt_abort) { + abort(); + } + } +} + +JEMALLOC_ALWAYS_INLINE void * +os_vm_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, + bool *commit) { + assert(alloc_size >= leadsize + size); + void *ret = (void *)((byte_t *)addr + leadsize); + size_t trailsize = alloc_size - leadsize - size; + + (void)commit; + if (leadsize != 0) { + os_vm_release(addr, leadsize); + } + if (trailsize != 0) { + os_vm_release((void *)((byte_t *)ret + size), trailsize); + } + return ret; +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_commit_impl(void *addr, size_t size, bool commit) { + assert(PAGE_ADDR2BASE(addr) == addr); + assert(PAGE_CEILING(size) == size); + + int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; + void *result = mmap( + addr, size, prot, mmap_flags | MAP_FIXED, PAGES_FD_TAG, 0); + if (result == MAP_FAILED) { + return true; + } + if (result != addr) { + /* + * We succeeded in mapping memory, but not in the right + * place. + */ + os_vm_release(result, size); + return true; + } + return false; +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_commit(void *addr, size_t size) { + return os_vm_commit_impl(addr, size, true); +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_decommit(void *addr, size_t size) { + return os_vm_commit_impl(addr, size, false); +} + +JEMALLOC_ALWAYS_INLINE void +os_vm_mark_guards(void *head, void *tail) { + assert(head != NULL || tail != NULL); + assert( + head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); +#ifdef JEMALLOC_HAVE_MPROTECT + if (head != NULL) { + mprotect(head, PAGE, PROT_NONE); + } + if (tail != NULL) { + mprotect(tail, PAGE, PROT_NONE); + } +#else + /* Decommit sets the region to PROT_NONE. */ + if (head != NULL) { + os_vm_decommit(head, PAGE); + } + if (tail != NULL) { + os_vm_decommit(tail, PAGE); + } +#endif +} + +JEMALLOC_ALWAYS_INLINE void +os_vm_unmark_guards(void *head, void *tail) { + assert(head != NULL || tail != NULL); + assert( + head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); +#ifdef JEMALLOC_HAVE_MPROTECT + bool head_and_tail = (head != NULL) && (tail != NULL); + size_t range = head_and_tail ? (uintptr_t)tail - (uintptr_t)head + PAGE + : SIZE_T_MAX; + /* + * The amount of work that the kernel does in mprotect depends on the + * range argument. SC_LARGE_MINCLASS is an arbitrary threshold chosen + * to prevent kernel from doing too much work that would outweigh the + * savings of performing one less system call. + */ + bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS; + if (ranged_mprotect) { + mprotect(head, range, PROT_READ | PROT_WRITE); + } else { + if (head != NULL) { + mprotect(head, PAGE, PROT_READ | PROT_WRITE); + } + if (tail != NULL) { + mprotect(tail, PAGE, PROT_READ | PROT_WRITE); + } + } +#else + if (head != NULL) { + os_vm_commit(head, PAGE); + } + if (tail != NULL) { + os_vm_commit(tail, PAGE); + } +#endif +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_purge_lazy(void *addr, size_t size) { +#if defined(JEMALLOC_PURGE_MADVISE_FREE) + return (madvise(addr, size, +# ifdef MADV_FREE + MADV_FREE +# else + JEMALLOC_MADV_FREE +# endif + ) + != 0); +#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \ + && !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) + return (madvise(addr, size, MADV_DONTNEED) != 0); +#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) \ + && !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS) + return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0); +#else + (void)addr; + (void)size; + not_reached(); +#endif +} + +#endif /* JEMALLOC_INTERNAL_OS_POSIX_VM_H */ diff --git a/include/jemalloc/internal/os/vm.h b/include/jemalloc/internal/os/vm.h new file mode 100644 index 00000000..d558152d --- /dev/null +++ b/include/jemalloc/internal/os/vm.h @@ -0,0 +1,44 @@ +#ifndef JEMALLOC_INTERNAL_OS_VM_H +#define JEMALLOC_INTERNAL_OS_VM_H + +#include "jemalloc/internal/jemalloc_internal_externs.h" +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/malloc_io.h" +#include "jemalloc/internal/os/detect.h" +#include "jemalloc/internal/pages.h" +#include "jemalloc/internal/sc.h" + +/* + * VM interface: raw page-level reserve/commit/decommit/guard/trim primitives + * used by src/pages.c. Platform-specific fast paths (e.g. the FreeBSD + * MAP_EXCL reservation shortcut) stay in src/pages.c, since they are + * portable decisions layered on top of these primitives, not primitives + * themselves. + * Default: posix/. Override: Windows (VirtualAlloc/VirtualFree). + * + * State: `os_overcommits`, set by pages_boot(), read directly by the + * backends below. + */ +extern bool os_overcommits; + +/* Functions required for implementation in each backend. */ +JEMALLOC_ALWAYS_INLINE void *os_vm_reserve( + void *hint, size_t size, size_t alignment, bool *commit); +JEMALLOC_ALWAYS_INLINE void os_vm_release(void *addr, size_t size); +JEMALLOC_ALWAYS_INLINE void *os_vm_trim(void *addr, size_t alloc_size, + size_t leadsize, size_t size, bool *commit); +JEMALLOC_ALWAYS_INLINE bool os_vm_commit(void *addr, size_t size); +JEMALLOC_ALWAYS_INLINE bool os_vm_decommit(void *addr, size_t size); +JEMALLOC_ALWAYS_INLINE void os_vm_mark_guards(void *head, void *tail); +JEMALLOC_ALWAYS_INLINE void os_vm_unmark_guards(void *head, void *tail); +JEMALLOC_ALWAYS_INLINE bool os_vm_purge_lazy(void *addr, size_t size); + +#if defined(_WIN32) +# include "jemalloc/internal/os/windows/vm.h" +#elif defined(JEMALLOC_OS_POSIX) +# include "jemalloc/internal/os/posix/vm.h" +#else +# error "OS layer: no vm backend for this platform; add os//vm.h" +#endif + +#endif /* JEMALLOC_INTERNAL_OS_VM_H */ diff --git a/include/jemalloc/internal/os/windows/vm.h b/include/jemalloc/internal/os/windows/vm.h new file mode 100644 index 00000000..daf02e86 --- /dev/null +++ b/include/jemalloc/internal/os/windows/vm.h @@ -0,0 +1,115 @@ +#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_VM_H +#define JEMALLOC_INTERNAL_OS_WINDOWS_VM_H + +#include "jemalloc/internal/jemalloc_preamble.h" + +JEMALLOC_ALWAYS_INLINE void * +os_vm_reserve(void *hint, size_t size, size_t alignment, bool *commit) { + assert(ALIGNMENT_ADDR2BASE(hint, os_page) == hint); + assert(ALIGNMENT_CEILING(size, os_page) == size); + assert(size != 0); + (void)alignment; + + if (os_overcommits) { + *commit = true; + } + + void *ret; + /* + * If VirtualAlloc can't allocate at the given address when one is + * given, it fails and returns NULL. + */ + ret = VirtualAlloc(hint, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0), + PAGE_READWRITE); + assert(ret == NULL || (hint == NULL && ret != hint) + || (hint != NULL && ret == hint)); + return ret; +} + +JEMALLOC_ALWAYS_INLINE void +os_vm_release(void *addr, size_t size) { + assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); + assert(ALIGNMENT_CEILING(size, os_page) == size); + + if (VirtualFree(addr, 0, MEM_RELEASE) == 0) { + char buf[BUFERROR_BUF]; + + buferror(get_errno(), buf, sizeof(buf)); + malloc_printf(": Error in VirtualFree(): %s\n", buf); + if (opt_abort) { + abort(); + } + } +} + +JEMALLOC_ALWAYS_INLINE void * +os_vm_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, + bool *commit) { + assert(alloc_size >= leadsize + size); + void *ret = (void *)((byte_t *)addr + leadsize); + + os_vm_release(addr, alloc_size); + void *new_addr = os_vm_reserve(ret, size, PAGE, commit); + if (new_addr == ret) { + return ret; + } + if (new_addr != NULL) { + os_vm_release(new_addr, size); + } + return NULL; +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_commit_impl(void *addr, size_t size, bool commit) { + assert(PAGE_ADDR2BASE(addr) == addr); + assert(PAGE_CEILING(size) == size); + + return (commit + ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE)) + : (!VirtualFree(addr, size, MEM_DECOMMIT))); +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_commit(void *addr, size_t size) { + return os_vm_commit_impl(addr, size, true); +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_decommit(void *addr, size_t size) { + return os_vm_commit_impl(addr, size, false); +} + +JEMALLOC_ALWAYS_INLINE void +os_vm_mark_guards(void *head, void *tail) { + assert(head != NULL || tail != NULL); + assert( + head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); + /* Decommit sets the region to MEM_DECOMMIT. */ + if (head != NULL) { + os_vm_decommit(head, PAGE); + } + if (tail != NULL) { + os_vm_decommit(tail, PAGE); + } +} + +JEMALLOC_ALWAYS_INLINE void +os_vm_unmark_guards(void *head, void *tail) { + assert(head != NULL || tail != NULL); + assert( + head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); + if (head != NULL) { + os_vm_commit(head, PAGE); + } + if (tail != NULL) { + os_vm_commit(tail, PAGE); + } +} + +JEMALLOC_ALWAYS_INLINE bool +os_vm_purge_lazy(void *addr, size_t size) { + VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); + return false; +} + +#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_VM_H */ diff --git a/src/pages.c b/src/pages.c index ed802a0e..9c34515b 100644 --- a/src/pages.c +++ b/src/pages.c @@ -14,21 +14,6 @@ # include # endif #endif -#ifdef __NetBSD__ -# include /* ilog2 */ -#endif -#ifdef JEMALLOC_HAVE_VM_MAKE_TAG -# define PAGES_FD_TAG VM_MAKE_TAG(254U) -#else -# define PAGES_FD_TAG -1 -#endif -#if defined(JEMALLOC_HAVE_PRCTL) && defined(JEMALLOC_PAGEID) -# include -# ifndef PR_SET_VMA -# define PR_SET_VMA 0x53564d41 -# define PR_SET_VMA_ANON_NAME 0 -# endif -#endif /******************************************************************************/ /* Data. */ @@ -36,12 +21,10 @@ /* Actual operating system page size, detected during bootstrap, <= PAGE. */ size_t os_page; -#ifndef _WIN32 -# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE) -# define PAGES_PROT_DECOMMIT (PROT_NONE) -static int mmap_flags; -#endif -static bool os_overcommits; +/* Set here. Consumed by os_vm_reserve/os_vm_commit in os/posix/vm.h. */ +int mmap_flags; +/* Set here, consumed directly by the os/vm.h backends. */ +bool os_overcommits; const char *const thp_mode_names[] = { "default", "always", "never", "not supported"}; @@ -106,156 +89,12 @@ madvise_MADV_DONTNEED_zeroes_pages(void) { } #endif -#ifdef JEMALLOC_PAGEID -static int -os_page_id(void *addr, size_t size, const char *name) { -# ifdef JEMALLOC_HAVE_PRCTL - /* - * While parsing `/proc//maps` file, the block could appear as - * 7f4836000000-7f4836800000 rw-p 00000000 00:00 0 [anon:jemalloc_pg_overcommit]` - */ - int n; - assert(addr != NULL); - n = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)addr, size, - (uintptr_t)name); - assert(n == 0 || (n == -1 && get_errno() == EINVAL)); - return n; -# else - return 0; -# endif -} -#endif - -/******************************************************************************/ -/* - * Function prototypes for static functions that are referenced prior to - * definition. - */ - -static void os_pages_unmap(void *addr, size_t size); - /******************************************************************************/ static void * -os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) { - assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); - assert(ALIGNMENT_CEILING(size, os_page) == size); - assert(size != 0); - - if (os_overcommits) { - *commit = true; - } - - void *ret; -#ifdef _WIN32 - /* - * If VirtualAlloc can't allocate at the given address when one is - * given, it fails and returns NULL. - */ - ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0), - PAGE_READWRITE); -#else - /* - * We don't use MAP_FIXED here, because it can cause the *replacement* - * of existing mappings, and we only want to create new mappings. - */ - { - int flags = mmap_flags; -# ifdef __NetBSD__ - /* - * On NetBSD PAGE for a platform is defined to the - * maximum page size of all machine architectures - * for that platform, so that we can use the same - * binaries across all machine architectures. - */ - if (alignment > os_page || PAGE > os_page) { - unsigned int a = ilog2(MAX(alignment, PAGE)); - flags |= MAP_ALIGNED(a); - } -# endif - int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - - ret = mmap(addr, size, prot, flags, PAGES_FD_TAG, 0); - } - assert(ret != NULL); - - if (ret == MAP_FAILED) { - ret = NULL; - } else if (addr != NULL && ret != addr) { - /* - * We succeeded in mapping memory, but not in the right place. - */ - os_pages_unmap(ret, size); - ret = NULL; - } -#endif - assert(ret == NULL || (addr == NULL && ret != addr) - || (addr != NULL && ret == addr)); -#ifdef JEMALLOC_PAGEID - if (ret != NULL) { - os_page_id(ret, size, - os_overcommits ? "jemalloc_pg_overcommit" : "jemalloc_pg"); - } -#endif - return ret; -} - -static void * -os_pages_trim( +pages_trim( void *addr, size_t alloc_size, size_t leadsize, size_t size, bool *commit) { - void *ret = (void *)((byte_t *)addr + leadsize); - - assert(alloc_size >= leadsize + size); -#ifdef _WIN32 - os_pages_unmap(addr, alloc_size); - void *new_addr = os_pages_map(ret, size, PAGE, commit); - if (new_addr == ret) { - return ret; - } - if (new_addr != NULL) { - os_pages_unmap(new_addr, size); - } - return NULL; -#else - size_t trailsize = alloc_size - leadsize - size; - - if (leadsize != 0) { - os_pages_unmap(addr, leadsize); - } - if (trailsize != 0) { - os_pages_unmap((void *)((byte_t *)ret + size), trailsize); - } - return ret; -#endif -} - -static void -os_pages_unmap(void *addr, size_t size) { - assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); - assert(ALIGNMENT_CEILING(size, os_page) == size); - -#ifdef _WIN32 - if (VirtualFree(addr, 0, MEM_RELEASE) == 0) -#else - if (munmap(addr, size) == -1) -#endif - { - char buf[BUFERROR_BUF]; - - buferror(get_errno(), buf, sizeof(buf)); - malloc_printf( - ": Error in " -#ifdef _WIN32 - "VirtualFree" -#else - "munmap" -#endif - "(): %s\n", - buf); - if (opt_abort) { - abort(); - } - } + return os_vm_trim(addr, alloc_size, leadsize, size, commit); } static void * @@ -268,13 +107,13 @@ pages_map_slow(size_t size, size_t alignment, bool *commit) { void *ret; do { - void *pages = os_pages_map(NULL, alloc_size, alignment, commit); + void *pages = os_vm_reserve(NULL, alloc_size, alignment, commit); if (pages == NULL) { return NULL; } size_t leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) - (uintptr_t)pages; - ret = os_pages_trim(pages, alloc_size, leadsize, size, commit); + ret = pages_trim(pages, alloc_size, leadsize, size, commit); } while (ret == NULL); assert(ret != NULL); @@ -287,34 +126,8 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { assert(alignment >= PAGE); assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr); -#if defined(__FreeBSD__) && defined(MAP_EXCL) - /* - * FreeBSD has mechanisms both to mmap at specific address without - * touching existing mappings, and to mmap with specific alignment. - */ - { - if (os_overcommits) { - *commit = true; - } - - int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - int flags = mmap_flags; - - if (addr != NULL) { - flags |= MAP_FIXED | MAP_EXCL; - } else { - unsigned alignment_bits = ffs_zu(alignment); - assert(alignment_bits > 0); - flags |= MAP_ALIGNED(alignment_bits); - } - - void *ret = mmap(addr, size, prot, flags, -1, 0); - if (ret == MAP_FAILED) { - ret = NULL; - } - - return ret; - } +#ifdef OS_VM_HAS_FIXED_ALIGNED_RESERVE + return os_vm_reserve_aligned_fixed(addr, size, alignment, commit); #endif /* * Ideally, there would be a way to specify alignment to mmap() (like @@ -322,7 +135,7 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { * hard to efficiently create aligned mappings. The reliable, but * slow method is to create a mapping that is over-sized, then trim the * excess. However, that always results in one or two calls to - * os_pages_unmap(), and it can leave holes in the process's virtual + * os_vm_release(), and it can leave holes in the process's virtual * memory map if memory grows downward. * * Optimistically try mapping precisely the right amount before falling @@ -330,13 +143,13 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { * approach works most of the time. */ - void *ret = os_pages_map(addr, size, os_page, commit); + void *ret = os_vm_reserve(addr, size, os_page, commit); if (ret == NULL || ret == addr) { return ret; } assert(addr == NULL); if (ALIGNMENT_ADDR2OFFSET(ret, alignment) != 0) { - os_pages_unmap(ret, size); + os_vm_release(ret, size); return pages_map_slow(size, alignment, commit); } @@ -349,37 +162,7 @@ pages_unmap(void *addr, size_t size) { assert(PAGE_ADDR2BASE(addr) == addr); assert(PAGE_CEILING(size) == size); - os_pages_unmap(addr, size); -} - -static bool -os_pages_commit(void *addr, size_t size, bool commit) { - assert(PAGE_ADDR2BASE(addr) == addr); - assert(PAGE_CEILING(size) == size); - -#ifdef _WIN32 - return (commit - ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE)) - : (!VirtualFree(addr, size, MEM_DECOMMIT))); -#else - { - int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; - void *result = mmap( - addr, size, prot, mmap_flags | MAP_FIXED, PAGES_FD_TAG, 0); - if (result == MAP_FAILED) { - return true; - } - if (result != addr) { - /* - * We succeeded in mapping memory, but not in the right - * place. - */ - os_pages_unmap(result, size); - return true; - } - return false; - } -#endif + os_vm_release(addr, size); } static bool @@ -388,7 +171,7 @@ pages_commit_impl(void *addr, size_t size, bool commit) { return true; } - return os_pages_commit(addr, size, commit); + return commit ? os_vm_commit(addr, size) : os_vm_decommit(addr, size); } bool @@ -403,61 +186,12 @@ pages_decommit(void *addr, size_t size) { void pages_mark_guards(void *head, void *tail) { - assert(head != NULL || tail != NULL); - assert( - head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); -#ifdef JEMALLOC_HAVE_MPROTECT - if (head != NULL) { - mprotect(head, PAGE, PROT_NONE); - } - if (tail != NULL) { - mprotect(tail, PAGE, PROT_NONE); - } -#else - /* Decommit sets to PROT_NONE / MEM_DECOMMIT. */ - if (head != NULL) { - os_pages_commit(head, PAGE, false); - } - if (tail != NULL) { - os_pages_commit(tail, PAGE, false); - } -#endif + os_vm_mark_guards(head, tail); } void pages_unmark_guards(void *head, void *tail) { - assert(head != NULL || tail != NULL); - assert( - head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail); -#ifdef JEMALLOC_HAVE_MPROTECT - bool head_and_tail = (head != NULL) && (tail != NULL); - size_t range = head_and_tail ? (uintptr_t)tail - (uintptr_t)head + PAGE - : SIZE_T_MAX; - /* - * The amount of work that the kernel does in mprotect depends on the - * range argument. SC_LARGE_MINCLASS is an arbitrary threshold chosen - * to prevent kernel from doing too much work that would outweigh the - * savings of performing one less system call. - */ - bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS; - if (ranged_mprotect) { - mprotect(head, range, PROT_READ | PROT_WRITE); - } else { - if (head != NULL) { - mprotect(head, PAGE, PROT_READ | PROT_WRITE); - } - if (tail != NULL) { - mprotect(tail, PAGE, PROT_READ | PROT_WRITE); - } - } -#else - if (head != NULL) { - os_pages_commit(head, PAGE, true); - } - if (tail != NULL) { - os_pages_commit(tail, PAGE, true); - } -#endif + os_vm_unmark_guards(head, tail); } bool @@ -476,27 +210,7 @@ pages_purge_lazy(void *addr, size_t size) { return true; } -#ifdef _WIN32 - VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); - return false; -#elif defined(JEMALLOC_PURGE_MADVISE_FREE) - return (madvise(addr, size, -# ifdef MADV_FREE - MADV_FREE -# else - JEMALLOC_MADV_FREE -# endif - ) - != 0); -#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \ - && !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) - return (madvise(addr, size, MADV_DONTNEED) != 0); -#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) \ - && !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS) - return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0); -#else - not_reached(); -#endif + return os_vm_purge_lazy(addr, size); } bool @@ -932,7 +646,7 @@ pages_boot(void) { /* Detect lazy purge runtime support. */ if (pages_can_purge_lazy) { bool committed = false; - void *madv_free_page = os_pages_map( + void *madv_free_page = os_vm_reserve( NULL, PAGE, PAGE, &committed); if (madv_free_page == NULL) { return true; @@ -941,7 +655,7 @@ pages_boot(void) { if (pages_purge_lazy(madv_free_page, PAGE)) { pages_can_purge_lazy_runtime = false; } - os_pages_unmap(madv_free_page, PAGE); + os_vm_release(madv_free_page, PAGE); } #endif if (init_process_madvise()) {