mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-08-17 21:03:54 +03:00
Move boot and overcommit setup to the OS layer.
This commit also migrates some detection in configure.ac into OS dipatcher since they both rely on the same preprocessor OS flags.
This commit is contained in:
parent
4b55768fdc
commit
2c87eff72c
11 changed files with 326 additions and 197 deletions
|
|
@ -805,7 +805,6 @@ case "${host}" in
|
|||
*-*-freebsd*)
|
||||
JE_APPEND_VS(CPPFLAGS, -D_BSD_SOURCE)
|
||||
abi="elf"
|
||||
AC_DEFINE([JEMALLOC_SYSCTL_VM_OVERCOMMIT], [ ], [ ])
|
||||
force_lazy_lock="1"
|
||||
;;
|
||||
*-*-dragonfly*)
|
||||
|
|
@ -825,7 +824,6 @@ case "${host}" in
|
|||
glibc="0"
|
||||
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_HAS_ALLOCA_H], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_THREADED_INIT], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_C11_ATOMICS], [ ], [ ])
|
||||
force_tls="0"
|
||||
|
|
@ -840,7 +838,6 @@ case "${host}" in
|
|||
abi="elf"
|
||||
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_HAS_ALLOCA_H], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_THREADED_INIT], [ ], [ ])
|
||||
if test "${LG_SIZEOF_PTR}" = "3"; then
|
||||
default_retain="1"
|
||||
|
|
@ -854,7 +851,6 @@ case "${host}" in
|
|||
glibc="1"
|
||||
AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_HAS_ALLOCA_H], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_THREADED_INIT], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_USE_CXX_THROW], [ ], [ ])
|
||||
if test "${LG_SIZEOF_PTR}" = "3"; then
|
||||
|
|
@ -867,7 +863,6 @@ case "${host}" in
|
|||
JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE)
|
||||
abi="elf"
|
||||
AC_DEFINE([JEMALLOC_HAS_ALLOCA_H], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_SYSCTL_VM_OVERCOMMIT], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_THREADED_INIT], [ ], [ ])
|
||||
AC_DEFINE([JEMALLOC_USE_CXX_THROW], [ ], [ ])
|
||||
;;
|
||||
|
|
|
|||
|
|
@ -290,15 +290,6 @@
|
|||
*/
|
||||
#undef JEMALLOC_ZONE
|
||||
|
||||
/*
|
||||
* Methods for determining whether the OS overcommits.
|
||||
* JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY: Linux's
|
||||
* /proc/sys/vm.overcommit_memory file.
|
||||
* JEMALLOC_SYSCTL_VM_OVERCOMMIT: FreeBSD's vm.overcommit sysctl.
|
||||
*/
|
||||
#undef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
#undef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
|
||||
|
||||
/* Defined if madvise(2) is available. */
|
||||
#undef JEMALLOC_HAVE_MADVISE
|
||||
|
||||
|
|
|
|||
45
include/jemalloc/internal/os/freebsd/overcommit.h
Normal file
45
include/jemalloc/internal/os/freebsd/overcommit.h
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
#ifndef JEMALLOC_INTERNAL_OS_FREEBSD_OVERCOMMIT_H
|
||||
#define JEMALLOC_INTERNAL_OS_FREEBSD_OVERCOMMIT_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
|
||||
#include <sys/mman.h>
|
||||
#include <sys/sysctl.h>
|
||||
#ifdef __FreeBSD__
|
||||
# include <vm/vm_param.h>
|
||||
#endif
|
||||
|
||||
/* Defined in src/pages.c. */
|
||||
extern int mmap_flags;
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
os_overcommits_sysctl(void) {
|
||||
int vm_overcommit;
|
||||
size_t sz;
|
||||
|
||||
sz = sizeof(vm_overcommit);
|
||||
# if defined(__FreeBSD__) && defined(VM_OVERCOMMIT)
|
||||
int mib[2];
|
||||
|
||||
mib[0] = CTL_VM;
|
||||
mib[1] = VM_OVERCOMMIT;
|
||||
if (sysctl(mib, 2, &vm_overcommit, &sz, NULL, 0) != 0) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
# else
|
||||
if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
# endif
|
||||
|
||||
return ((vm_overcommit & 0x3) == 0);
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
os_overcommit_boot(void) {
|
||||
mmap_flags = MAP_PRIVATE | MAP_ANON;
|
||||
os_overcommits = os_overcommits_sysctl();
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_OS_FREEBSD_OVERCOMMIT_H */
|
||||
152
include/jemalloc/internal/os/linux/overcommit.h
Normal file
152
include/jemalloc/internal/os/linux/overcommit.h
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
#ifndef JEMALLOC_INTERNAL_OS_LINUX_OVERCOMMIT_H
|
||||
#define JEMALLOC_INTERNAL_OS_LINUX_OVERCOMMIT_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/malloc_io.h"
|
||||
|
||||
#include <sys/mman.h>
|
||||
|
||||
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
|
||||
/*
|
||||
* Set by os_overcommit_boot()'s madvise_MADV_DONTNEED_zeroes_pages() probe,
|
||||
* consumed by pages_purge_forced() in src/pages.c. Kept behind this macro
|
||||
* (rather than assumed unconditional, even though every configure.ac Linux
|
||||
* branch defines it today) so this file doesn't silently desync from
|
||||
* configure.ac if that ever changes.
|
||||
*
|
||||
* Defined in src/pages.c.
|
||||
*/
|
||||
extern int madvise_dont_need_zeros_is_faulty;
|
||||
#endif
|
||||
|
||||
/* Defined in src/pages.c. */
|
||||
extern int mmap_flags;
|
||||
|
||||
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
|
||||
/*
|
||||
* Check that MADV_DONTNEED will actually zero pages on subsequent access.
|
||||
*
|
||||
* Since qemu does not support this, yet [1], and you can get very tricky
|
||||
* assert if you will run program with jemalloc in use under qemu:
|
||||
*
|
||||
* <jemalloc>: ../contrib/jemalloc/src/extent.c:1195: Failed assertion: "p[i] == 0"
|
||||
*
|
||||
* [1]: https://patchwork.kernel.org/patch/10576637/
|
||||
*/
|
||||
JEMALLOC_ALWAYS_INLINE int
|
||||
madvise_MADV_DONTNEED_zeroes_pages(void) {
|
||||
size_t size = PAGE;
|
||||
|
||||
void *addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
|
||||
if (addr == MAP_FAILED) {
|
||||
malloc_write(
|
||||
"<jemalloc>: Cannot allocate memory for "
|
||||
"MADV_DONTNEED check\n");
|
||||
if (opt_abort) {
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
memset(addr, 'A', size);
|
||||
int works;
|
||||
if (madvise(addr, size, MADV_DONTNEED) == 0) {
|
||||
works = memchr(addr, 'A', size) == NULL;
|
||||
} else {
|
||||
/*
|
||||
* If madvise() does not support MADV_DONTNEED, then we can
|
||||
* call it anyway, and use it's return code.
|
||||
*/
|
||||
works = 1;
|
||||
}
|
||||
|
||||
if (munmap(addr, size) != 0) {
|
||||
malloc_write(
|
||||
"<jemalloc>: Cannot deallocate memory for "
|
||||
"MADV_DONTNEED check\n");
|
||||
if (opt_abort) {
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
return works;
|
||||
}
|
||||
#endif
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
os_overcommits_proc(void) {
|
||||
int fd;
|
||||
char buf[1];
|
||||
|
||||
# if defined(O_CLOEXEC)
|
||||
fd = malloc_open(
|
||||
"/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
|
||||
# else
|
||||
fd = malloc_open("/proc/sys/vm/overcommit_memory", O_RDONLY);
|
||||
if (fd != -1) {
|
||||
fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
|
||||
}
|
||||
# endif
|
||||
|
||||
if (fd == -1) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
|
||||
ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf));
|
||||
malloc_close(fd);
|
||||
|
||||
if (nread < 1) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
/*
|
||||
* /proc/sys/vm/overcommit_memory meanings:
|
||||
* 0: Heuristic overcommit.
|
||||
* 1: Always overcommit.
|
||||
* 2: Never overcommit.
|
||||
*/
|
||||
return (buf[0] == '0' || buf[0] == '1');
|
||||
}
|
||||
|
||||
/*
|
||||
* Bundles what pages_boot() used to do after the page-size check, on Linux:
|
||||
* 1. DONTNEED-zeros probe (madvise_MADV_DONTNEED_zeroes_pages()).
|
||||
* 2. mmap_flags assembly (MAP_PRIVATE | MAP_ANON, plus MAP_NORESERVE when
|
||||
* the kernel overcommits).
|
||||
* 3. Overcommit detection via /proc/sys/vm/overcommit_memory
|
||||
* (os_overcommits_proc() above).
|
||||
*/
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
os_overcommit_boot(void) {
|
||||
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
|
||||
if (!opt_trust_madvise) {
|
||||
madvise_dont_need_zeros_is_faulty =
|
||||
!madvise_MADV_DONTNEED_zeroes_pages();
|
||||
if (madvise_dont_need_zeros_is_faulty) {
|
||||
malloc_write(
|
||||
"<jemalloc>: MADV_DONTNEED does not work (memset will be used instead)\n");
|
||||
malloc_write(
|
||||
"<jemalloc>: (This is the expected behaviour if you are running under QEMU)\n");
|
||||
}
|
||||
} else {
|
||||
/*
|
||||
* In case opt_trust_madvise is disable,
|
||||
* do not do runtime check.
|
||||
*/
|
||||
madvise_dont_need_zeros_is_faulty = 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
mmap_flags = MAP_PRIVATE | MAP_ANON;
|
||||
|
||||
os_overcommits = os_overcommits_proc();
|
||||
#ifdef MAP_NORESERVE
|
||||
if (os_overcommits) {
|
||||
mmap_flags |= MAP_NORESERVE;
|
||||
}
|
||||
#endif
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_OS_LINUX_OVERCOMMIT_H */
|
||||
47
include/jemalloc/internal/os/overcommit.h
Normal file
47
include/jemalloc/internal/os/overcommit.h
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
#ifndef JEMALLOC_INTERNAL_OS_OVERCOMMIT_H
|
||||
#define JEMALLOC_INTERNAL_OS_OVERCOMMIT_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/pages.h"
|
||||
|
||||
/*
|
||||
* Overcommit interface: boot-time detection of kernel memory-policy facts
|
||||
* that src/pages.c and os/vm.h's reserve/commit primitives depend on --
|
||||
* whether the kernel overcommits, and (Linux only) whether MADV_DONTNEED is
|
||||
* known to zero pages and what mmap() flags to use. Separate from os/vm.h
|
||||
* because none of this is a VM reserve/commit primitive itself; it's boot-
|
||||
* time policy detection that some of those primitives happen to read.
|
||||
*
|
||||
* Unlike every other os/<module>.h, this one has FOUR tiers instead of the
|
||||
* usual two (posix/windows) or three (posix/windows/darwin): FreeBSD and
|
||||
* Linux each have substantial, disjoint, dedicated detection logic (real
|
||||
* sysctl/proc-file probing, not a one-line difference), so each gets its own
|
||||
* file with no internal #ifdef __FreeBSD__/__linux__ branching -- unlike
|
||||
* os/darwin/cpu.h (which duplicates two mostly-identical functions just to
|
||||
* override a third), splitting these out duplicates nothing, since
|
||||
* os_overcommits_sysctl() and os_overcommits_proc()/the DONTNEED-zeros probe
|
||||
* never had anything in common to begin with. Default: posix/ (NetBSD
|
||||
* hardcoded true, everyone else false -- matches this project's
|
||||
* pre-refactor behavior for all of them). Overrides: Windows (never
|
||||
* overcommits), FreeBSD/kFreeBSD (sysctl), Linux (/proc/sys/vm).
|
||||
*
|
||||
* State: `os_overcommits`, set by os_overcommit_boot(), read by os/vm.h's
|
||||
* backends. Defined in src/pages.c.
|
||||
*/
|
||||
extern bool os_overcommits;
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool os_overcommit_boot(void);
|
||||
|
||||
#if defined(_WIN32)
|
||||
# include "jemalloc/internal/os/windows/overcommit.h"
|
||||
#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
|
||||
# include "jemalloc/internal/os/freebsd/overcommit.h"
|
||||
#elif defined(__linux__)
|
||||
# include "jemalloc/internal/os/linux/overcommit.h"
|
||||
#elif defined(JEMALLOC_OS_POSIX)
|
||||
# include "jemalloc/internal/os/posix/overcommit.h"
|
||||
#else
|
||||
# error "OS layer: no overcommit backend for this platform; add os/<os>/overcommit.h"
|
||||
#endif
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_OS_OVERCOMMIT_H */
|
||||
22
include/jemalloc/internal/os/posix/overcommit.h
Normal file
22
include/jemalloc/internal/os/posix/overcommit.h
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
#ifndef JEMALLOC_INTERNAL_OS_POSIX_OVERCOMMIT_H
|
||||
#define JEMALLOC_INTERNAL_OS_POSIX_OVERCOMMIT_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
|
||||
#include <sys/mman.h>
|
||||
|
||||
/* Defined in src/pages.c. */
|
||||
extern int mmap_flags;
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
os_overcommit_boot(void) {
|
||||
mmap_flags = MAP_PRIVATE | MAP_ANON;
|
||||
#ifdef __NetBSD__
|
||||
os_overcommits = true;
|
||||
#else
|
||||
os_overcommits = false;
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_OS_POSIX_OVERCOMMIT_H */
|
||||
|
|
@ -2,6 +2,8 @@
|
|||
#define JEMALLOC_INTERNAL_OS_POSIX_VM_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/malloc_io.h"
|
||||
#include "jemalloc/internal/os/overcommit.h"
|
||||
|
||||
#include "jemalloc/internal/bit_util.h"
|
||||
|
||||
|
|
@ -25,10 +27,6 @@
|
|||
#define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
|
||||
#define PAGES_PROT_DECOMMIT (PROT_NONE)
|
||||
|
||||
/* mmap flags assembled by pages_boot(); MAP_PRIVATE | MAP_ANON (+ MAP_NORESERVE
|
||||
* when the kernel overcommits). */
|
||||
extern int mmap_flags;
|
||||
|
||||
#ifdef JEMALLOC_PAGEID
|
||||
JEMALLOC_ALWAYS_INLINE int
|
||||
os_page_id(void *addr, size_t size, const char *name) {
|
||||
|
|
@ -287,4 +285,21 @@ os_vm_purge_lazy(void *addr, size_t size) {
|
|||
#endif
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
os_vm_page_size(void) {
|
||||
#ifdef __FreeBSD__
|
||||
/*
|
||||
* This returns the value obtained from
|
||||
* the auxv vector, avoiding a syscall.
|
||||
*/
|
||||
return getpagesize();
|
||||
#else
|
||||
long result = sysconf(_SC_PAGESIZE);
|
||||
if (result == -1) {
|
||||
return PAGE;
|
||||
}
|
||||
return (size_t)result;
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_OS_POSIX_VM_H */
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/malloc_io.h"
|
||||
#include "jemalloc/internal/os/detect.h"
|
||||
#include "jemalloc/internal/os/overcommit.h"
|
||||
#include "jemalloc/internal/pages.h"
|
||||
#include "jemalloc/internal/sc.h"
|
||||
|
||||
|
|
@ -16,10 +17,10 @@
|
|||
* themselves.
|
||||
* Default: posix/. Override: Windows (VirtualAlloc/VirtualFree).
|
||||
*
|
||||
* State: `os_overcommits`, set by pages_boot(), read directly by the
|
||||
* backends below.
|
||||
* `os_overcommits` (read by os_vm_reserve/os_vm_commit_impl below) is
|
||||
* declared and set by os/overcommit.h, a separate module -- boot-time
|
||||
* kernel-policy detection isn't itself a VM reserve/commit primitive.
|
||||
*/
|
||||
extern bool os_overcommits;
|
||||
|
||||
/* Functions required for implementation in each backend. */
|
||||
JEMALLOC_ALWAYS_INLINE void *os_vm_reserve(
|
||||
|
|
@ -32,6 +33,7 @@ JEMALLOC_ALWAYS_INLINE bool os_vm_decommit(void *addr, size_t size);
|
|||
JEMALLOC_ALWAYS_INLINE void os_vm_mark_guards(void *head, void *tail);
|
||||
JEMALLOC_ALWAYS_INLINE void os_vm_unmark_guards(void *head, void *tail);
|
||||
JEMALLOC_ALWAYS_INLINE bool os_vm_purge_lazy(void *addr, size_t size);
|
||||
JEMALLOC_ALWAYS_INLINE size_t os_vm_page_size(void);
|
||||
|
||||
#if defined(_WIN32)
|
||||
# include "jemalloc/internal/os/windows/vm.h"
|
||||
|
|
|
|||
20
include/jemalloc/internal/os/windows/overcommit.h
Normal file
20
include/jemalloc/internal/os/windows/overcommit.h
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
#ifndef JEMALLOC_INTERNAL_OS_WINDOWS_OVERCOMMIT_H
|
||||
#define JEMALLOC_INTERNAL_OS_WINDOWS_OVERCOMMIT_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE bool
|
||||
os_overcommit_boot(void) {
|
||||
/*
|
||||
* Windows never overcommits, so this just hardcodes os_overcommits =
|
||||
* false. There's no sysctl/proc-file probe to run (contrast with
|
||||
* os_overcommits_sysctl() in os/freebsd/overcommit.h and
|
||||
* os_overcommits_proc() in os/linux/overcommit.h) and no
|
||||
* DONTNEED-zeros probe or mmap_flags to assemble (both
|
||||
* POSIX/madvise-only concepts).
|
||||
*/
|
||||
os_overcommits = false;
|
||||
return false;
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_OVERCOMMIT_H */
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
#define JEMALLOC_INTERNAL_OS_WINDOWS_VM_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/os/overcommit.h"
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE void *
|
||||
os_vm_reserve(void *hint, size_t size, size_t alignment, bool *commit) {
|
||||
|
|
@ -112,4 +113,11 @@ os_vm_purge_lazy(void *addr, size_t size) {
|
|||
return false;
|
||||
}
|
||||
|
||||
JEMALLOC_ALWAYS_INLINE size_t
|
||||
os_vm_page_size(void) {
|
||||
SYSTEM_INFO si;
|
||||
GetSystemInfo(&si);
|
||||
return (size_t)si.dwPageSize;
|
||||
}
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_OS_WINDOWS_VM_H */
|
||||
|
|
|
|||
184
src/pages.c
184
src/pages.c
|
|
@ -8,22 +8,15 @@
|
|||
#include "jemalloc/internal/pages.h"
|
||||
#include "jemalloc/internal/sc.h"
|
||||
|
||||
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
# include <sys/sysctl.h>
|
||||
# ifdef __FreeBSD__
|
||||
# include <vm/vm_param.h>
|
||||
# endif
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
/* Data. */
|
||||
|
||||
/* Actual operating system page size, detected during bootstrap, <= PAGE. */
|
||||
size_t os_page;
|
||||
|
||||
/* Set here. Consumed by os_vm_reserve/os_vm_commit in os/posix/vm.h. */
|
||||
/* Set here. Consumed by os_vm_reserve/os_vm_commit_impl in os/posix/vm.h. */
|
||||
int mmap_flags;
|
||||
/* Set here, consumed directly by the os/vm.h backends. */
|
||||
/* Set here, consumed directly by the os/overcommit.h and os/vm.h backends. */
|
||||
bool os_overcommits;
|
||||
|
||||
const char *const thp_mode_names[] = {
|
||||
|
|
@ -37,56 +30,9 @@ system_thp_mode_t init_system_thp_mode;
|
|||
static bool pages_can_purge_lazy_runtime = true;
|
||||
|
||||
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
|
||||
static int madvise_dont_need_zeros_is_faulty = -1;
|
||||
/**
|
||||
* Check that MADV_DONTNEED will actually zero pages on subsequent access.
|
||||
*
|
||||
* Since qemu does not support this, yet [1], and you can get very tricky
|
||||
* assert if you will run program with jemalloc in use under qemu:
|
||||
*
|
||||
* <jemalloc>: ../contrib/jemalloc/src/extent.c:1195: Failed assertion: "p[i] == 0"
|
||||
*
|
||||
* [1]: https://patchwork.kernel.org/patch/10576637/
|
||||
*/
|
||||
static int
|
||||
madvise_MADV_DONTNEED_zeroes_pages(void) {
|
||||
size_t size = PAGE;
|
||||
|
||||
void *addr = mmap(NULL, size, PROT_READ | PROT_WRITE,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
|
||||
|
||||
if (addr == MAP_FAILED) {
|
||||
malloc_write(
|
||||
"<jemalloc>: Cannot allocate memory for "
|
||||
"MADV_DONTNEED check\n");
|
||||
if (opt_abort) {
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
memset(addr, 'A', size);
|
||||
int works;
|
||||
if (madvise(addr, size, MADV_DONTNEED) == 0) {
|
||||
works = memchr(addr, 'A', size) == NULL;
|
||||
} else {
|
||||
/*
|
||||
* If madvise() does not support MADV_DONTNEED, then we can
|
||||
* call it anyway, and use it's return code.
|
||||
*/
|
||||
works = 1;
|
||||
}
|
||||
|
||||
if (munmap(addr, size) != 0) {
|
||||
malloc_write(
|
||||
"<jemalloc>: Cannot deallocate memory for "
|
||||
"MADV_DONTNEED check\n");
|
||||
if (opt_abort) {
|
||||
abort();
|
||||
}
|
||||
}
|
||||
|
||||
return works;
|
||||
}
|
||||
/* Set by os_overcommit_boot()'s madvise_MADV_DONTNEED_zeroes_pages() probe
|
||||
* (os/posix/overcommit.h), consumed by pages_purge_forced() below. */
|
||||
int madvise_dont_need_zeros_is_faulty = -1;
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
|
|
@ -414,88 +360,6 @@ pages_purge_process_madvise(void *vec, size_t vec_len, size_t total_bytes) {
|
|||
return pages_purge_process_madvise_impl(vec, vec_len, total_bytes);
|
||||
}
|
||||
|
||||
static size_t
|
||||
os_page_detect(void) {
|
||||
#ifdef _WIN32
|
||||
SYSTEM_INFO si;
|
||||
GetSystemInfo(&si);
|
||||
return si.dwPageSize;
|
||||
#elif defined(__FreeBSD__)
|
||||
/*
|
||||
* This returns the value obtained from
|
||||
* the auxv vector, avoiding a syscall.
|
||||
*/
|
||||
return getpagesize();
|
||||
#else
|
||||
long result = sysconf(_SC_PAGESIZE);
|
||||
if (result == -1) {
|
||||
return PAGE;
|
||||
}
|
||||
return (size_t)result;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
static bool
|
||||
os_overcommits_sysctl(void) {
|
||||
int vm_overcommit;
|
||||
size_t sz;
|
||||
|
||||
sz = sizeof(vm_overcommit);
|
||||
# if defined(__FreeBSD__) && defined(VM_OVERCOMMIT)
|
||||
int mib[2];
|
||||
|
||||
mib[0] = CTL_VM;
|
||||
mib[1] = VM_OVERCOMMIT;
|
||||
if (sysctl(mib, 2, &vm_overcommit, &sz, NULL, 0) != 0) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
# else
|
||||
if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
# endif
|
||||
|
||||
return ((vm_overcommit & 0x3) == 0);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
|
||||
static bool
|
||||
os_overcommits_proc(void) {
|
||||
int fd;
|
||||
char buf[1];
|
||||
|
||||
# if defined(O_CLOEXEC)
|
||||
fd = malloc_open(
|
||||
"/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC);
|
||||
# else
|
||||
fd = malloc_open("/proc/sys/vm/overcommit_memory", O_RDONLY);
|
||||
if (fd != -1) {
|
||||
fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
|
||||
}
|
||||
# endif
|
||||
|
||||
if (fd == -1) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
|
||||
ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf));
|
||||
malloc_close(fd);
|
||||
|
||||
if (nread < 1) {
|
||||
return false; /* Error. */
|
||||
}
|
||||
/*
|
||||
* /proc/sys/vm/overcommit_memory meanings:
|
||||
* 0: Heuristic overcommit.
|
||||
* 1: Always overcommit.
|
||||
* 2: Never overcommit.
|
||||
*/
|
||||
return (buf[0] == '0' || buf[0] == '1');
|
||||
}
|
||||
#endif
|
||||
|
||||
static bool
|
||||
pages_should_skip_set_thp_state(void) {
|
||||
if (opt_thp == thp_mode_do_nothing
|
||||
|
|
@ -591,7 +455,7 @@ label_error:
|
|||
|
||||
bool
|
||||
pages_boot(void) {
|
||||
os_page = os_page_detect();
|
||||
os_page = os_vm_page_size();
|
||||
if (os_page > PAGE) {
|
||||
malloc_write("<jemalloc>: Unsupported system page size\n");
|
||||
if (opt_abort) {
|
||||
|
|
@ -600,41 +464,9 @@ pages_boot(void) {
|
|||
return true;
|
||||
}
|
||||
|
||||
#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS
|
||||
if (!opt_trust_madvise) {
|
||||
madvise_dont_need_zeros_is_faulty =
|
||||
!madvise_MADV_DONTNEED_zeroes_pages();
|
||||
if (madvise_dont_need_zeros_is_faulty) {
|
||||
malloc_write(
|
||||
"<jemalloc>: MADV_DONTNEED does not work (memset will be used instead)\n");
|
||||
malloc_write(
|
||||
"<jemalloc>: (This is the expected behaviour if you are running under QEMU)\n");
|
||||
}
|
||||
} else {
|
||||
/* In case opt_trust_madvise is disable,
|
||||
* do not do runtime check */
|
||||
madvise_dont_need_zeros_is_faulty = 0;
|
||||
if (os_overcommit_boot()) {
|
||||
return true;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifndef _WIN32
|
||||
mmap_flags = MAP_PRIVATE | MAP_ANON;
|
||||
#endif
|
||||
|
||||
#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
|
||||
os_overcommits = os_overcommits_sysctl();
|
||||
#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
|
||||
os_overcommits = os_overcommits_proc();
|
||||
# ifdef MAP_NORESERVE
|
||||
if (os_overcommits) {
|
||||
mmap_flags |= MAP_NORESERVE;
|
||||
}
|
||||
# endif
|
||||
#elif defined(__NetBSD__)
|
||||
os_overcommits = true;
|
||||
#else
|
||||
os_overcommits = false;
|
||||
#endif
|
||||
|
||||
init_thp_state();
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue