romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-08-17 21:03:54 +03:00
Code Issues Projects Releases Packages Wiki Activity

Move VM reserve/release/purge/commit/decommit/guards/trim to OS layer

Browse source 
Migrates some VM functions namely, reserve, release, purge, commit,
decommit, guards (mark and unmark), and trim to OS layer.
This commit is contained in:
guangli-dai 2026-07-27 20:12:33 -07:00 • committed by Guangli Dai
parent 89b9aef8f7
commit c8e2e0134c
5 changed files with 472 additions and 306 deletions
Download patch file Download diff file Expand all files Collapse all files

3
include/jemalloc/internal/os.h
View file

@ -46,4 +46,7 @@
/* CPU */
#include "jemalloc/internal/os/cpu.h"
/* VM */
#include "jemalloc/internal/os/vm.h"
#endif /* JEMALLOC_INTERNAL_OS_H */

290
include/jemalloc/internal/os/posix/vm.h Normal file
View file

@ -0,0 +1,290 @@
#ifndef JEMALLOC_INTERNAL_OS_POSIX_VM_H
#define JEMALLOC_INTERNAL_OS_POSIX_VM_H
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/bit_util.h"
#include <sys/mman.h>
#if defined(JEMALLOC_HAVE_PRCTL) && defined(JEMALLOC_PAGEID)
# include <sys/prctl.h>
# ifndef PR_SET_VMA
# define PR_SET_VMA 0x53564d41
# define PR_SET_VMA_ANON_NAME 0
# endif
#endif
#ifdef __NetBSD__
# include <sys/bitops.h> /* ilog2 */
#endif
#ifdef JEMALLOC_HAVE_VM_MAKE_TAG
# define PAGES_FD_TAG VM_MAKE_TAG(254U)
#else
# define PAGES_FD_TAG -1
#endif
#define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
#define PAGES_PROT_DECOMMIT (PROT_NONE)
/* mmap flags assembled by pages_boot(); MAP_PRIVATE | MAP_ANON (+ MAP_NORESERVE
* when the kernel overcommits). */
extern int mmap_flags;
#ifdef JEMALLOC_PAGEID
JEMALLOC_ALWAYS_INLINE int
os_page_id(void *addr, size_t size, const char *name) {
# ifdef JEMALLOC_HAVE_PRCTL
/*
* While parsing `/proc/<pid>/maps` file, the block could appear as
* 7f4836000000-7f4836800000 rw-p 00000000 00:00 0 [anon:jemalloc_pg_overcommit]`
*/
int n;
assert(addr != NULL);
n = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)addr, size,
(uintptr_t)name);
assert(n == 0 || (n == -1 && get_errno() == EINVAL));
return n;
# else
return 0;
# endif
}
#endif
JEMALLOC_ALWAYS_INLINE void *
os_vm_reserve(void *hint, size_t size, size_t alignment, bool *commit) {
assert(ALIGNMENT_ADDR2BASE(hint, os_page) == hint);
assert(ALIGNMENT_CEILING(size, os_page) == size);
assert(size != 0);
if (os_overcommits) {
*commit = true;
}
void *ret;
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
{
int flags = mmap_flags;
#ifdef __NetBSD__
/*
* On NetBSD PAGE for a platform is defined to the
* maximum page size of all machine architectures
* for that platform, so that we can use the same
* binaries across all machine architectures.
*/
if (alignment > os_page || PAGE > os_page) {
unsigned int a = ilog2(MAX(alignment, PAGE));
flags |= MAP_ALIGNED(a);
}
#endif
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
ret = mmap(hint, size, prot, flags, PAGES_FD_TAG, 0);
}
assert(ret != NULL);
if (ret == MAP_FAILED) {
ret = NULL;
} else if (hint != NULL && ret != hint) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
os_vm_release(ret, size);
ret = NULL;
}
assert(ret == NULL || (hint == NULL && ret != hint)
|| (hint != NULL && ret == hint));
#ifdef JEMALLOC_PAGEID
if (ret != NULL) {
os_page_id(ret, size,
os_overcommits ? "jemalloc_pg_overcommit" : "jemalloc_pg");
}
#endif
return ret;
}
#if defined(__FreeBSD__) && defined(MAP_EXCL)
/*
* FreeBSD has mechanisms both to mmap at a specific address without
* touching existing mappings (MAP_EXCL), and to mmap with a specific
* alignment.
*/
# define OS_VM_HAS_FIXED_ALIGNED_RESERVE
JEMALLOC_ALWAYS_INLINE void *
os_vm_reserve_aligned_fixed(void *addr, size_t size, size_t alignment,
bool *commit) {
if (os_overcommits) {
*commit = true;
}
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
int flags = mmap_flags;
if (addr != NULL) {
flags |= MAP_FIXED | MAP_EXCL;
} else {
unsigned alignment_bits = ffs_zu(alignment);
assert(alignment_bits > 0);
flags |= MAP_ALIGNED(alignment_bits);
}
void *ret = mmap(addr, size, prot, flags, -1, 0);
if (ret == MAP_FAILED) {
ret = NULL;
}
return ret;
}
#endif
JEMALLOC_ALWAYS_INLINE void
os_vm_release(void *addr, size_t size) {
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
assert(ALIGNMENT_CEILING(size, os_page) == size);
if (munmap(addr, size) == -1) {
char buf[BUFERROR_BUF];
buferror(get_errno(), buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in munmap(): %s\n", buf);
if (opt_abort) {
abort();
}
}
}
JEMALLOC_ALWAYS_INLINE void *
os_vm_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
bool *commit) {
assert(alloc_size >= leadsize + size);
void *ret = (void *)((byte_t *)addr + leadsize);
size_t trailsize = alloc_size - leadsize - size;
(void)commit;
if (leadsize != 0) {
os_vm_release(addr, leadsize);
}
if (trailsize != 0) {
os_vm_release((void *)((byte_t *)ret + size), trailsize);
}
return ret;
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_commit_impl(void *addr, size_t size, bool commit) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
void *result = mmap(
addr, size, prot, mmap_flags | MAP_FIXED, PAGES_FD_TAG, 0);
if (result == MAP_FAILED) {
return true;
}
if (result != addr) {
/*
* We succeeded in mapping memory, but not in the right
* place.
*/
os_vm_release(result, size);
return true;
}
return false;
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_commit(void *addr, size_t size) {
return os_vm_commit_impl(addr, size, true);
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_decommit(void *addr, size_t size) {
return os_vm_commit_impl(addr, size, false);
}
JEMALLOC_ALWAYS_INLINE void
os_vm_mark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(
head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
if (head != NULL) {
mprotect(head, PAGE, PROT_NONE);
}
if (tail != NULL) {
mprotect(tail, PAGE, PROT_NONE);
}
#else
/* Decommit sets the region to PROT_NONE. */
if (head != NULL) {
os_vm_decommit(head, PAGE);
}
if (tail != NULL) {
os_vm_decommit(tail, PAGE);
}
#endif
}
JEMALLOC_ALWAYS_INLINE void
os_vm_unmark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(
head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
bool head_and_tail = (head != NULL) && (tail != NULL);
size_t range = head_and_tail ? (uintptr_t)tail - (uintptr_t)head + PAGE
: SIZE_T_MAX;
/*
* The amount of work that the kernel does in mprotect depends on the
* range argument. SC_LARGE_MINCLASS is an arbitrary threshold chosen
* to prevent kernel from doing too much work that would outweigh the
* savings of performing one less system call.
*/
bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS;
if (ranged_mprotect) {
mprotect(head, range, PROT_READ | PROT_WRITE);
} else {
if (head != NULL) {
mprotect(head, PAGE, PROT_READ | PROT_WRITE);
}
if (tail != NULL) {
mprotect(tail, PAGE, PROT_READ | PROT_WRITE);
}
}
#else
if (head != NULL) {
os_vm_commit(head, PAGE);
}
if (tail != NULL) {
os_vm_commit(tail, PAGE);
}
#endif
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_purge_lazy(void *addr, size_t size) {
#if defined(JEMALLOC_PURGE_MADVISE_FREE)
return (madvise(addr, size,
# ifdef MADV_FREE
MADV_FREE
# else
JEMALLOC_MADV_FREE
# endif
)
!= 0);
#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \
&& !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
return (madvise(addr, size, MADV_DONTNEED) != 0);
#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) \
&& !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
#else
(void)addr;
(void)size;
not_reached();
#endif
}
#endif /* JEMALLOC_INTERNAL_OS_POSIX_VM_H */

44
include/jemalloc/internal/os/vm.h Normal file
View file

@ -0,0 +1,44 @@
#ifndef JEMALLOC_INTERNAL_OS_VM_H
#define JEMALLOC_INTERNAL_OS_VM_H
#include "jemalloc/internal/jemalloc_internal_externs.h"
#include "jemalloc/internal/jemalloc_preamble.h"
#include "jemalloc/internal/malloc_io.h"
#include "jemalloc/internal/os/detect.h"
#include "jemalloc/internal/pages.h"
#include "jemalloc/internal/sc.h"
/*
* VM interface: raw page-level reserve/commit/decommit/guard/trim primitives
* used by src/pages.c. Platform-specific fast paths (e.g. the FreeBSD
* MAP_EXCL reservation shortcut) stay in src/pages.c, since they are
* portable decisions layered on top of these primitives, not primitives
* themselves.
* Default: posix/. Override: Windows (VirtualAlloc/VirtualFree).
*
* State: `os_overcommits`, set by pages_boot(), read directly by the
* backends below.
*/
extern bool os_overcommits;
/* Functions required for implementation in each backend. */
JEMALLOC_ALWAYS_INLINE void *os_vm_reserve(
void *hint, size_t size, size_t alignment, bool *commit);
JEMALLOC_ALWAYS_INLINE void os_vm_release(void *addr, size_t size);
JEMALLOC_ALWAYS_INLINE void *os_vm_trim(void *addr, size_t alloc_size,
size_t leadsize, size_t size, bool *commit);
JEMALLOC_ALWAYS_INLINE bool os_vm_commit(void *addr, size_t size);
JEMALLOC_ALWAYS_INLINE bool os_vm_decommit(void *addr, size_t size);
JEMALLOC_ALWAYS_INLINE void os_vm_mark_guards(void *head, void *tail);
JEMALLOC_ALWAYS_INLINE void os_vm_unmark_guards(void *head, void *tail);
JEMALLOC_ALWAYS_INLINE bool os_vm_purge_lazy(void *addr, size_t size);
#if defined(_WIN32)
# include "jemalloc/internal/os/windows/vm.h"
#elif defined(JEMALLOC_OS_POSIX)
# include "jemalloc/internal/os/posix/vm.h"
#else
# error "OS layer: no vm backend for this platform; add os/<os>/vm.h"
#endif
#endif /* JEMALLOC_INTERNAL_OS_VM_H */

115
include/jemalloc/internal/os/windows/vm.h Normal file
View file

@ -0,0 +1,115 @@
#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_VM_H
#define JEMALLOC_INTERNAL_OS_WINDOWS_VM_H
#include "jemalloc/internal/jemalloc_preamble.h"
JEMALLOC_ALWAYS_INLINE void *
os_vm_reserve(void *hint, size_t size, size_t alignment, bool *commit) {
assert(ALIGNMENT_ADDR2BASE(hint, os_page) == hint);
assert(ALIGNMENT_CEILING(size, os_page) == size);
assert(size != 0);
(void)alignment;
if (os_overcommits) {
*commit = true;
}
void *ret;
/*
* If VirtualAlloc can't allocate at the given address when one is
* given, it fails and returns NULL.
*/
ret = VirtualAlloc(hint, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
PAGE_READWRITE);
assert(ret == NULL || (hint == NULL && ret != hint)
|| (hint != NULL && ret == hint));
return ret;
}
JEMALLOC_ALWAYS_INLINE void
os_vm_release(void *addr, size_t size) {
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
assert(ALIGNMENT_CEILING(size, os_page) == size);
if (VirtualFree(addr, 0, MEM_RELEASE) == 0) {
char buf[BUFERROR_BUF];
buferror(get_errno(), buf, sizeof(buf));
malloc_printf("<jemalloc>: Error in VirtualFree(): %s\n", buf);
if (opt_abort) {
abort();
}
}
}
JEMALLOC_ALWAYS_INLINE void *
os_vm_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
bool *commit) {
assert(alloc_size >= leadsize + size);
void *ret = (void *)((byte_t *)addr + leadsize);
os_vm_release(addr, alloc_size);
void *new_addr = os_vm_reserve(ret, size, PAGE, commit);
if (new_addr == ret) {
return ret;
}
if (new_addr != NULL) {
os_vm_release(new_addr, size);
}
return NULL;
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_commit_impl(void *addr, size_t size, bool commit) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
return (commit
? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE))
: (!VirtualFree(addr, size, MEM_DECOMMIT)));
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_commit(void *addr, size_t size) {
return os_vm_commit_impl(addr, size, true);
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_decommit(void *addr, size_t size) {
return os_vm_commit_impl(addr, size, false);
}
JEMALLOC_ALWAYS_INLINE void
os_vm_mark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(
head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
/* Decommit sets the region to MEM_DECOMMIT. */
if (head != NULL) {
os_vm_decommit(head, PAGE);
}
if (tail != NULL) {
os_vm_decommit(tail, PAGE);
}
}
JEMALLOC_ALWAYS_INLINE void
os_vm_unmark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(
head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
if (head != NULL) {
os_vm_commit(head, PAGE);
}
if (tail != NULL) {
os_vm_commit(tail, PAGE);
}
}
JEMALLOC_ALWAYS_INLINE bool
os_vm_purge_lazy(void *addr, size_t size) {
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
return false;
}
#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_VM_H */

326
src/pages.c
View file

@ -14,21 +14,6 @@
# include <vm/vm_param.h>
# endif
#endif
#ifdef __NetBSD__
# include <sys/bitops.h> /* ilog2 */
#endif
#ifdef JEMALLOC_HAVE_VM_MAKE_TAG
# define PAGES_FD_TAG VM_MAKE_TAG(254U)
#else
# define PAGES_FD_TAG -1
#endif
#if defined(JEMALLOC_HAVE_PRCTL) && defined(JEMALLOC_PAGEID)
# include <sys/prctl.h>
# ifndef PR_SET_VMA
# define PR_SET_VMA 0x53564d41
# define PR_SET_VMA_ANON_NAME 0
# endif
#endif
/******************************************************************************/
/* Data. */
@ -36,12 +21,10 @@
/* Actual operating system page size, detected during bootstrap, <= PAGE. */
size_t os_page;
#ifndef _WIN32
# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
# define PAGES_PROT_DECOMMIT (PROT_NONE)
static int mmap_flags;
#endif
static bool os_overcommits;
/* Set here. Consumed by os_vm_reserve/os_vm_commit in os/posix/vm.h. */
int mmap_flags;
/* Set here, consumed directly by the os/vm.h backends. */
bool os_overcommits;
const char *const thp_mode_names[] = {
"default", "always", "never", "not supported"};
@ -106,156 +89,12 @@ madvise_MADV_DONTNEED_zeroes_pages(void) {
}
#endif
#ifdef JEMALLOC_PAGEID
static int
os_page_id(void *addr, size_t size, const char *name) {
# ifdef JEMALLOC_HAVE_PRCTL
/*
* While parsing `/proc/<pid>/maps` file, the block could appear as
* 7f4836000000-7f4836800000 rw-p 00000000 00:00 0 [anon:jemalloc_pg_overcommit]`
*/
int n;
assert(addr != NULL);
n = prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, (uintptr_t)addr, size,
(uintptr_t)name);
assert(n == 0 || (n == -1 && get_errno() == EINVAL));
return n;
# else
return 0;
# endif
}
#endif
/******************************************************************************/
/*
* Function prototypes for static functions that are referenced prior to
* definition.
*/
static void os_pages_unmap(void *addr, size_t size);
/******************************************************************************/
static void *
os_pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
assert(ALIGNMENT_CEILING(size, os_page) == size);
assert(size != 0);
if (os_overcommits) {
*commit = true;
}
void *ret;
#ifdef _WIN32
/*
* If VirtualAlloc can't allocate at the given address when one is
* given, it fails and returns NULL.
*/
ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
PAGE_READWRITE);
#else
/*
* We don't use MAP_FIXED here, because it can cause the *replacement*
* of existing mappings, and we only want to create new mappings.
*/
{
int flags = mmap_flags;
# ifdef __NetBSD__
/*
* On NetBSD PAGE for a platform is defined to the
* maximum page size of all machine architectures
* for that platform, so that we can use the same
* binaries across all machine architectures.
*/
if (alignment > os_page || PAGE > os_page) {
unsigned int a = ilog2(MAX(alignment, PAGE));
flags |= MAP_ALIGNED(a);
}
# endif
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
ret = mmap(addr, size, prot, flags, PAGES_FD_TAG, 0);
}
assert(ret != NULL);
if (ret == MAP_FAILED) {
ret = NULL;
} else if (addr != NULL && ret != addr) {
/*
* We succeeded in mapping memory, but not in the right place.
*/
os_pages_unmap(ret, size);
ret = NULL;
}
#endif
assert(ret == NULL || (addr == NULL && ret != addr)
|| (addr != NULL && ret == addr));
#ifdef JEMALLOC_PAGEID
if (ret != NULL) {
os_page_id(ret, size,
os_overcommits ? "jemalloc_pg_overcommit" : "jemalloc_pg");
}
#endif
return ret;
}
static void *
os_pages_trim(
pages_trim(
void *addr, size_t alloc_size, size_t leadsize, size_t size, bool *commit) {
void *ret = (void *)((byte_t *)addr + leadsize);
assert(alloc_size >= leadsize + size);
#ifdef _WIN32
os_pages_unmap(addr, alloc_size);
void *new_addr = os_pages_map(ret, size, PAGE, commit);
if (new_addr == ret) {
return ret;
}
if (new_addr != NULL) {
os_pages_unmap(new_addr, size);
}
return NULL;
#else
size_t trailsize = alloc_size - leadsize - size;
if (leadsize != 0) {
os_pages_unmap(addr, leadsize);
}
if (trailsize != 0) {
os_pages_unmap((void *)((byte_t *)ret + size), trailsize);
}
return ret;
#endif
}
static void
os_pages_unmap(void *addr, size_t size) {
assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr);
assert(ALIGNMENT_CEILING(size, os_page) == size);
#ifdef _WIN32
if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
#else
if (munmap(addr, size) == -1)
#endif
{
char buf[BUFERROR_BUF];
buferror(get_errno(), buf, sizeof(buf));
malloc_printf(
"<jemalloc>: Error in "
#ifdef _WIN32
"VirtualFree"
#else
"munmap"
#endif
"(): %s\n",
buf);
if (opt_abort) {
abort();
}
}
return os_vm_trim(addr, alloc_size, leadsize, size, commit);
}
static void *
@ -268,13 +107,13 @@ pages_map_slow(size_t size, size_t alignment, bool *commit) {
void *ret;
do {
void *pages = os_pages_map(NULL, alloc_size, alignment, commit);
void *pages = os_vm_reserve(NULL, alloc_size, alignment, commit);
if (pages == NULL) {
return NULL;
}
size_t leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment)
- (uintptr_t)pages;
ret = os_pages_trim(pages, alloc_size, leadsize, size, commit);
ret = pages_trim(pages, alloc_size, leadsize, size, commit);
} while (ret == NULL);
assert(ret != NULL);
@ -287,34 +126,8 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
assert(alignment >= PAGE);
assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr);
#if defined(__FreeBSD__) && defined(MAP_EXCL)
/*
* FreeBSD has mechanisms both to mmap at specific address without
* touching existing mappings, and to mmap with specific alignment.
*/
{
if (os_overcommits) {
*commit = true;
}
int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
int flags = mmap_flags;
if (addr != NULL) {
flags |= MAP_FIXED | MAP_EXCL;
} else {
unsigned alignment_bits = ffs_zu(alignment);
assert(alignment_bits > 0);
flags |= MAP_ALIGNED(alignment_bits);
}
void *ret = mmap(addr, size, prot, flags, -1, 0);
if (ret == MAP_FAILED) {
ret = NULL;
}
return ret;
}
#ifdef OS_VM_HAS_FIXED_ALIGNED_RESERVE
return os_vm_reserve_aligned_fixed(addr, size, alignment, commit);
#endif
/*
* Ideally, there would be a way to specify alignment to mmap() (like
@ -322,7 +135,7 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
* hard to efficiently create aligned mappings. The reliable, but
* slow method is to create a mapping that is over-sized, then trim the
* excess. However, that always results in one or two calls to
* os_pages_unmap(), and it can leave holes in the process's virtual
* os_vm_release(), and it can leave holes in the process's virtual
* memory map if memory grows downward.
*
* Optimistically try mapping precisely the right amount before falling
@ -330,13 +143,13 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) {
* approach works most of the time.
*/
void *ret = os_pages_map(addr, size, os_page, commit);
void *ret = os_vm_reserve(addr, size, os_page, commit);
if (ret == NULL || ret == addr) {
return ret;
}
assert(addr == NULL);
if (ALIGNMENT_ADDR2OFFSET(ret, alignment) != 0) {
os_pages_unmap(ret, size);
os_vm_release(ret, size);
return pages_map_slow(size, alignment, commit);
}
@ -349,37 +162,7 @@ pages_unmap(void *addr, size_t size) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
os_pages_unmap(addr, size);
}
static bool
os_pages_commit(void *addr, size_t size, bool commit) {
assert(PAGE_ADDR2BASE(addr) == addr);
assert(PAGE_CEILING(size) == size);
#ifdef _WIN32
return (commit
? (addr != VirtualAlloc(addr, size, MEM_COMMIT, PAGE_READWRITE))
: (!VirtualFree(addr, size, MEM_DECOMMIT)));
#else
{
int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
void *result = mmap(
addr, size, prot, mmap_flags | MAP_FIXED, PAGES_FD_TAG, 0);
if (result == MAP_FAILED) {
return true;
}
if (result != addr) {
/*
* We succeeded in mapping memory, but not in the right
* place.
*/
os_pages_unmap(result, size);
return true;
}
return false;
}
#endif
os_vm_release(addr, size);
}
static bool
@ -388,7 +171,7 @@ pages_commit_impl(void *addr, size_t size, bool commit) {
return true;
}
return os_pages_commit(addr, size, commit);
return commit ? os_vm_commit(addr, size) : os_vm_decommit(addr, size);
}
bool
@ -403,61 +186,12 @@ pages_decommit(void *addr, size_t size) {
void
pages_mark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(
head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
if (head != NULL) {
mprotect(head, PAGE, PROT_NONE);
}
if (tail != NULL) {
mprotect(tail, PAGE, PROT_NONE);
}
#else
/* Decommit sets to PROT_NONE / MEM_DECOMMIT. */
if (head != NULL) {
os_pages_commit(head, PAGE, false);
}
if (tail != NULL) {
os_pages_commit(tail, PAGE, false);
}
#endif
os_vm_mark_guards(head, tail);
}
void
pages_unmark_guards(void *head, void *tail) {
assert(head != NULL || tail != NULL);
assert(
head == NULL || tail == NULL || (uintptr_t)head < (uintptr_t)tail);
#ifdef JEMALLOC_HAVE_MPROTECT
bool head_and_tail = (head != NULL) && (tail != NULL);
size_t range = head_and_tail ? (uintptr_t)tail - (uintptr_t)head + PAGE
: SIZE_T_MAX;
/*
* The amount of work that the kernel does in mprotect depends on the
* range argument. SC_LARGE_MINCLASS is an arbitrary threshold chosen
* to prevent kernel from doing too much work that would outweigh the
* savings of performing one less system call.
*/
bool ranged_mprotect = head_and_tail && range <= SC_LARGE_MINCLASS;
if (ranged_mprotect) {
mprotect(head, range, PROT_READ | PROT_WRITE);
} else {
if (head != NULL) {
mprotect(head, PAGE, PROT_READ | PROT_WRITE);
}
if (tail != NULL) {
mprotect(tail, PAGE, PROT_READ | PROT_WRITE);
}
}
#else
if (head != NULL) {
os_pages_commit(head, PAGE, true);
}
if (tail != NULL) {
os_pages_commit(tail, PAGE, true);
}
#endif
os_vm_unmark_guards(head, tail);
}
bool
@ -476,27 +210,7 @@ pages_purge_lazy(void *addr, size_t size) {
return true;
}
#ifdef _WIN32
VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
return false;
#elif defined(JEMALLOC_PURGE_MADVISE_FREE)
return (madvise(addr, size,
# ifdef MADV_FREE
MADV_FREE
# else
JEMALLOC_MADV_FREE
# endif
)
!= 0);
#elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) \
&& !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS)
return (madvise(addr, size, MADV_DONTNEED) != 0);
#elif defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED) \
&& !defined(JEMALLOC_PURGE_POSIX_MADVISE_DONTNEED_ZEROS)
return (posix_madvise(addr, size, POSIX_MADV_DONTNEED) != 0);
#else
not_reached();
#endif
return os_vm_purge_lazy(addr, size);
}
bool
@ -932,7 +646,7 @@ pages_boot(void) {
/* Detect lazy purge runtime support. */
if (pages_can_purge_lazy) {
bool committed = false;
void *madv_free_page = os_pages_map(
void *madv_free_page = os_vm_reserve(
NULL, PAGE, PAGE, &committed);
if (madv_free_page == NULL) {
return true;
@ -941,7 +655,7 @@ pages_boot(void) {
if (pages_purge_lazy(madv_free_page, PAGE)) {
pages_can_purge_lazy_runtime = false;
}
os_pages_unmap(madv_free_page, PAGE);
os_vm_release(madv_free_page, PAGE);
}
#endif
if (init_process_madvise()) {