mirror of
https://github.com/jemalloc/jemalloc.git
synced 2026-04-14 14:41:42 +03:00
[EASY] Extract hpa_central component from hpa source file
This commit is contained in:
Slobodan Predolac
Guangli Dai
parent
355774270d
commit
8a06b086f3
13 changed files with 184 additions and 145 deletions
|
|
@ -123,6 +123,7 @@ C_SRCS := $(srcroot)src/jemalloc.c \
|
|||
$(srcroot)src/san_bump.c \
|
||||
$(srcroot)src/hook.c \
|
||||
$(srcroot)src/hpa.c \
|
||||
$(srcroot)src/hpa_central.c \
|
||||
$(srcroot)src/hpa_hooks.c \
|
||||
$(srcroot)src/hpa_utils.c \
|
||||
$(srcroot)src/hpdata.c \
|
||||
|
|
|
|||
|
|
@ -6,36 +6,13 @@
|
|||
#include "jemalloc/internal/edata_cache.h"
|
||||
#include "jemalloc/internal/emap.h"
|
||||
#include "jemalloc/internal/exp_grow.h"
|
||||
#include "jemalloc/internal/hpa_central.h"
|
||||
#include "jemalloc/internal/hpa_hooks.h"
|
||||
#include "jemalloc/internal/hpa_opts.h"
|
||||
#include "jemalloc/internal/mutex.h"
|
||||
#include "jemalloc/internal/pai.h"
|
||||
#include "jemalloc/internal/psset.h"
|
||||
|
||||
typedef struct hpa_central_s hpa_central_t;
|
||||
struct hpa_central_s {
|
||||
/*
|
||||
* Guards expansion of eden. We separate this from the regular mutex so
|
||||
* that cheaper operations can still continue while we're doing the OS
|
||||
* call.
|
||||
*/
|
||||
malloc_mutex_t grow_mtx;
|
||||
/*
|
||||
* Either NULL (if empty), or some integer multiple of a
|
||||
* hugepage-aligned number of hugepages. We carve them off one at a
|
||||
* time to satisfy new pageslab requests.
|
||||
*
|
||||
* Guarded by grow_mtx.
|
||||
*/
|
||||
void *eden;
|
||||
size_t eden_len;
|
||||
/* Source for metadata. */
|
||||
base_t *base;
|
||||
|
||||
/* The HPA hooks. */
|
||||
hpa_hooks_t hooks;
|
||||
};
|
||||
|
||||
typedef struct hpa_shard_nonderived_stats_s hpa_shard_nonderived_stats_t;
|
||||
struct hpa_shard_nonderived_stats_s {
|
||||
/*
|
||||
|
|
@ -165,8 +142,6 @@ bool hpa_hugepage_size_exceeds_limit(void);
|
|||
* just that it can function properly given the system it's running on.
|
||||
*/
|
||||
bool hpa_supported(void);
|
||||
bool hpa_central_init(
|
||||
hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
|
||||
bool hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
|
||||
base_t *base, edata_cache_t *edata_cache, unsigned ind,
|
||||
const hpa_shard_opts_t *opts);
|
||||
|
|
|
|||
41
include/jemalloc/internal/hpa_central.h
Normal file
41
include/jemalloc/internal/hpa_central.h
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
#ifndef JEMALLOC_INTERNAL_HPA_CENTRAL_H
|
||||
#define JEMALLOC_INTERNAL_HPA_CENTRAL_H
|
||||
|
||||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/base.h"
|
||||
#include "jemalloc/internal/hpa_hooks.h"
|
||||
#include "jemalloc/internal/hpdata.h"
|
||||
#include "jemalloc/internal/mutex.h"
|
||||
#include "jemalloc/internal/tsd_types.h"
|
||||
|
||||
typedef struct hpa_central_s hpa_central_t;
|
||||
struct hpa_central_s {
|
||||
/*
|
||||
* Guards expansion of eden. We separate this from the regular mutex so
|
||||
* that cheaper operations can still continue while we're doing the OS
|
||||
* call.
|
||||
*/
|
||||
malloc_mutex_t grow_mtx;
|
||||
/*
|
||||
* Either NULL (if empty), or some integer multiple of a
|
||||
* hugepage-aligned number of hugepages. We carve them off one at a
|
||||
* time to satisfy new pageslab requests.
|
||||
*
|
||||
* Guarded by grow_mtx.
|
||||
*/
|
||||
void *eden;
|
||||
size_t eden_len;
|
||||
/* Source for metadata. */
|
||||
base_t *base;
|
||||
|
||||
/* The HPA hooks. */
|
||||
hpa_hooks_t hooks;
|
||||
};
|
||||
|
||||
bool hpa_central_init(
|
||||
hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks);
|
||||
|
||||
hpdata_t *hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
|
||||
uint64_t age, bool hugify_eager, bool *oom);
|
||||
|
||||
#endif /* JEMALLOC_INTERNAL_HPA_CENTRAL_H */
|
||||
|
|
@ -61,6 +61,7 @@
|
|||
<ClCompile Include="..\..\..\..\src\fxp.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hook.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpdata.c" />
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@
|
|||
<ClCompile Include="..\..\..\..\src\hpa.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
|
|
@ -206,4 +209,4 @@
|
|||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@
|
|||
<ClCompile Include="..\..\..\..\src\fxp.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hook.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpdata.c" />
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@
|
|||
<ClCompile Include="..\..\..\..\src\hpa.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
|
|
@ -206,4 +209,4 @@
|
|||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@
|
|||
<ClCompile Include="..\..\..\..\src\fxp.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hook.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpdata.c" />
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@
|
|||
<ClCompile Include="..\..\..\..\src\hpa.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
|
|
@ -206,4 +209,4 @@
|
|||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@
|
|||
<ClCompile Include="..\..\..\..\src\fxp.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hook.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpa_utils.c" />
|
||||
<ClCompile Include="..\..\..\..\src\hpdata.c" />
|
||||
|
|
|
|||
|
|
@ -67,6 +67,9 @@
|
|||
<ClCompile Include="..\..\..\..\src\hpa.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_central.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
<ClCompile Include="..\..\..\..\src\hpa_hooks.c">
|
||||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
|
|
@ -206,4 +209,4 @@
|
|||
<Filter>Source Files</Filter>
|
||||
</ClCompile>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
</Project>
|
||||
|
|
|
|||
115
src/hpa.c
115
src/hpa.c
|
|
@ -8,8 +8,6 @@
|
|||
#include "jemalloc/internal/witness.h"
|
||||
#include "jemalloc/internal/jemalloc_probe.h"
|
||||
|
||||
#define HPA_EDEN_SIZE (128 * HUGEPAGE)
|
||||
|
||||
static edata_t *hpa_alloc(tsdn_t *tsdn, pai_t *self, size_t size,
|
||||
size_t alignment, bool zero, bool guarded, bool frequent_reuse,
|
||||
bool *deferred_work_generated);
|
||||
|
|
@ -75,119 +73,6 @@ hpa_do_consistency_checks(hpa_shard_t *shard) {
|
|||
assert(shard->base != NULL);
|
||||
}
|
||||
|
||||
bool
|
||||
hpa_central_init(
|
||||
hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
|
||||
/* malloc_conf processing should have filtered out these cases. */
|
||||
assert(hpa_supported());
|
||||
bool err;
|
||||
err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow",
|
||||
WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
|
||||
if (err) {
|
||||
return true;
|
||||
}
|
||||
|
||||
central->base = base;
|
||||
central->eden = NULL;
|
||||
central->eden_len = 0;
|
||||
central->hooks = *hooks;
|
||||
return false;
|
||||
}
|
||||
|
||||
static hpdata_t *
|
||||
hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
|
||||
return (hpdata_t *)base_alloc(
|
||||
tsdn, central->base, sizeof(hpdata_t), CACHELINE);
|
||||
}
|
||||
|
||||
static hpdata_t *
|
||||
hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
|
||||
uint64_t age, bool hugify_eager, bool *oom) {
|
||||
/* Don't yet support big allocations; these should get filtered out. */
|
||||
assert(size <= HUGEPAGE);
|
||||
/*
|
||||
* Should only try to extract from the central allocator if the local
|
||||
* shard is exhausted. We should hold the grow_mtx on that shard.
|
||||
*/
|
||||
witness_assert_positive_depth_to_rank(
|
||||
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
|
||||
|
||||
malloc_mutex_lock(tsdn, ¢ral->grow_mtx);
|
||||
*oom = false;
|
||||
|
||||
hpdata_t *ps = NULL;
|
||||
bool start_as_huge = hugify_eager
|
||||
|| (init_system_thp_mode == system_thp_mode_always
|
||||
&& opt_experimental_hpa_start_huge_if_thp_always);
|
||||
|
||||
/* Is eden a perfect fit? */
|
||||
if (central->eden != NULL && central->eden_len == HUGEPAGE) {
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
hpdata_init(ps, central->eden, age, start_as_huge);
|
||||
central->eden = NULL;
|
||||
central->eden_len = 0;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return ps;
|
||||
}
|
||||
|
||||
/*
|
||||
* We're about to try to allocate from eden by splitting. If eden is
|
||||
* NULL, we have to allocate it too. Otherwise, we just have to
|
||||
* allocate an edata_t for the new psset.
|
||||
*/
|
||||
if (central->eden == NULL) {
|
||||
/* Allocate address space, bailing if we fail. */
|
||||
void *new_eden = central->hooks.map(HPA_EDEN_SIZE);
|
||||
if (new_eden == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
if (hugify_eager) {
|
||||
central->hooks.hugify(
|
||||
new_eden, HPA_EDEN_SIZE, /* sync */ false);
|
||||
}
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
central->hooks.unmap(new_eden, HPA_EDEN_SIZE);
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
central->eden = new_eden;
|
||||
central->eden_len = HPA_EDEN_SIZE;
|
||||
} else {
|
||||
/* Eden is already nonempty; only need an edata for ps. */
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
assert(ps != NULL);
|
||||
assert(central->eden != NULL);
|
||||
assert(central->eden_len > HUGEPAGE);
|
||||
assert(central->eden_len % HUGEPAGE == 0);
|
||||
assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
|
||||
|
||||
hpdata_init(ps, central->eden, age, start_as_huge);
|
||||
|
||||
char *eden_char = (char *)central->eden;
|
||||
eden_char += HUGEPAGE;
|
||||
central->eden = (void *)eden_char;
|
||||
central->eden_len -= HUGEPAGE;
|
||||
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
|
||||
return ps;
|
||||
}
|
||||
|
||||
bool
|
||||
hpa_shard_init(hpa_shard_t *shard, hpa_central_t *central, emap_t *emap,
|
||||
base_t *base, edata_cache_t *edata_cache, unsigned ind,
|
||||
|
|
|
|||
121
src/hpa_central.c
Normal file
121
src/hpa_central.c
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
#include "jemalloc/internal/jemalloc_preamble.h"
|
||||
#include "jemalloc/internal/jemalloc_internal_includes.h"
|
||||
|
||||
#include "jemalloc/internal/hpa_central.h"
|
||||
#include "jemalloc/internal/tsd.h"
|
||||
#include "jemalloc/internal/witness.h"
|
||||
|
||||
#define HPA_EDEN_SIZE (128 * HUGEPAGE)
|
||||
|
||||
bool
|
||||
hpa_central_init(
|
||||
hpa_central_t *central, base_t *base, const hpa_hooks_t *hooks) {
|
||||
/* malloc_conf processing should have filtered out these cases. */
|
||||
assert(hpa_supported());
|
||||
bool err;
|
||||
err = malloc_mutex_init(¢ral->grow_mtx, "hpa_central_grow",
|
||||
WITNESS_RANK_HPA_CENTRAL_GROW, malloc_mutex_rank_exclusive);
|
||||
if (err) {
|
||||
return true;
|
||||
}
|
||||
|
||||
central->base = base;
|
||||
central->eden = NULL;
|
||||
central->eden_len = 0;
|
||||
central->hooks = *hooks;
|
||||
return false;
|
||||
}
|
||||
|
||||
static hpdata_t *
|
||||
hpa_alloc_ps(tsdn_t *tsdn, hpa_central_t *central) {
|
||||
return (hpdata_t *)base_alloc(
|
||||
tsdn, central->base, sizeof(hpdata_t), CACHELINE);
|
||||
}
|
||||
|
||||
hpdata_t *
|
||||
hpa_central_extract(tsdn_t *tsdn, hpa_central_t *central, size_t size,
|
||||
uint64_t age, bool hugify_eager, bool *oom) {
|
||||
/* Don't yet support big allocations; these should get filtered out. */
|
||||
assert(size <= HUGEPAGE);
|
||||
/*
|
||||
* Should only try to extract from the central allocator if the local
|
||||
* shard is exhausted. We should hold the grow_mtx on that shard.
|
||||
*/
|
||||
witness_assert_positive_depth_to_rank(
|
||||
tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_HPA_SHARD_GROW);
|
||||
|
||||
malloc_mutex_lock(tsdn, ¢ral->grow_mtx);
|
||||
*oom = false;
|
||||
|
||||
hpdata_t *ps = NULL;
|
||||
bool start_as_huge = hugify_eager
|
||||
|| (init_system_thp_mode == system_thp_mode_always
|
||||
&& opt_experimental_hpa_start_huge_if_thp_always);
|
||||
|
||||
/* Is eden a perfect fit? */
|
||||
if (central->eden != NULL && central->eden_len == HUGEPAGE) {
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
hpdata_init(ps, central->eden, age, start_as_huge);
|
||||
central->eden = NULL;
|
||||
central->eden_len = 0;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return ps;
|
||||
}
|
||||
|
||||
/*
|
||||
* We're about to try to allocate from eden by splitting. If eden is
|
||||
* NULL, we have to allocate it too. Otherwise, we just have to
|
||||
* allocate an edata_t for the new psset.
|
||||
*/
|
||||
if (central->eden == NULL) {
|
||||
/* Allocate address space, bailing if we fail. */
|
||||
void *new_eden = central->hooks.map(HPA_EDEN_SIZE);
|
||||
if (new_eden == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
if (hugify_eager) {
|
||||
central->hooks.hugify(
|
||||
new_eden, HPA_EDEN_SIZE, /* sync */ false);
|
||||
}
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
central->hooks.unmap(new_eden, HPA_EDEN_SIZE);
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
central->eden = new_eden;
|
||||
central->eden_len = HPA_EDEN_SIZE;
|
||||
} else {
|
||||
/* Eden is already nonempty; only need an edata for ps. */
|
||||
ps = hpa_alloc_ps(tsdn, central);
|
||||
if (ps == NULL) {
|
||||
*oom = true;
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
assert(ps != NULL);
|
||||
assert(central->eden != NULL);
|
||||
assert(central->eden_len > HUGEPAGE);
|
||||
assert(central->eden_len % HUGEPAGE == 0);
|
||||
assert(HUGEPAGE_ADDR2BASE(central->eden) == central->eden);
|
||||
|
||||
hpdata_init(ps, central->eden, age, start_as_huge);
|
||||
|
||||
char *eden_char = (char *)central->eden;
|
||||
eden_char += HUGEPAGE;
|
||||
central->eden = (void *)eden_char;
|
||||
central->eden_len -= HUGEPAGE;
|
||||
|
||||
malloc_mutex_unlock(tsdn, ¢ral->grow_mtx);
|
||||
|
||||
return ps;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue