romenskiy2012/jemalloc
1
0
Fork 0
mirror of https://github.com/jemalloc/jemalloc.git synced 2026-07-28 03:13:09 +03:00
Code Issues Projects Releases Packages Wiki Activity

Use SEC in PAC to reduce lock contention on the ecaches

Browse source 
Add a small extent cache in front of the PAC ecaches. Allocs and dallocs
that fit are served from per-shard SEC bins without taking the ecache
mutex; overflow falls through to the backing ecaches, including
ecache_pinned for pinned extents.

The feature is gated behind experimental_pac_sec_nshards (default 0,
disabled). To support independent HPA and PAC SEC instances,
sec_alloc/sec_dalloc/sec_fill take an explicit shard argument, with HPA
and PAC using separate TSD shard slots.
This commit is contained in:
Bin Liu 2026-05-19 00:11:15 -07:00
parent 8aa7e5ec03
commit ccf9cdf417
19 changed files with 986 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

38
src/hpa.c
View file

@ -14,6 +14,18 @@ const char *const hpa_hugify_style_names[] = {"auto", "none", "eager", "lazy"};
bool opt_experimental_hpa_start_huge_if_thp_always = true;
bool opt_experimental_hpa_enforce_hugify = false;
static inline uint8_t
hpa_sec_shard_pick(tsdn_t *tsdn, sec_t *sec) {
if (sec->opts.nshards <= 1) {
return 0;
}
if (tsdn_null(tsdn)) {
return 0;
}
tsd_t *tsd = tsdn_tsd(tsdn);
return sec_shard_pick(tsd, sec, tsd_sec_shardp_get(tsd));
}
JET_EXTERN bool
hpa_hugepage_size_exceeds_limit(void) {
return HUGEPAGE > HUGEPAGE_MAX_EXPECTED_SIZE;
@ -945,9 +957,13 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
&& (size > shard->opts.slab_max_alloc)) {
return NULL;
}
edata_t *edata = sec_alloc(tsdn, &shard->sec, size);
if (edata != NULL) {
return edata;
edata_t *edata = NULL;
if (sec_size_supported(&shard->sec, size)) {
edata = sec_alloc(tsdn, &shard->sec, size,
hpa_sec_shard_pick(tsdn, &shard->sec));
if (edata != NULL) {
return edata;
}
}
edata_list_active_t results;
edata_list_active_init(&results);
@ -966,7 +982,8 @@ hpa_alloc(tsdn_t *tsdn, hpa_shard_t *shard, size_t size, size_t alignment,
}
if (nsuccess > 0) {
assert(sec_size_supported(&shard->sec, size));
sec_fill(tsdn, &shard->sec, size, &results, nsuccess);
sec_fill(tsdn, &shard->sec, size, &results, nsuccess,
hpa_sec_shard_pick(tsdn, &shard->sec));
/* Unlikely rollback in case of overfill */
if (!edata_list_active_empty(&results)) {
hpa_dalloc_batch(
@ -1059,11 +1076,14 @@ hpa_dalloc(tsdn_t *tsdn, hpa_shard_t *shard, edata_t *edata,
edata_list_active_init(&dalloc_list);
edata_list_active_append(&dalloc_list, edata);
sec_dalloc(tsdn, &shard->sec, &dalloc_list);
if (edata_list_active_empty(&dalloc_list)) {
/* sec consumed the pointer */
*deferred_work_generated = false;
return;
if (sec_size_supported(&shard->sec, edata_size_get(edata))) {
sec_dalloc(tsdn, &shard->sec, &dalloc_list,
hpa_sec_shard_pick(tsdn, &shard->sec));
if (edata_list_active_empty(&dalloc_list)) {
/* sec consumed the pointer */
*deferred_work_generated = false;
return;
}
}
/* We may have more than one pointer to flush now */
hpa_dalloc_batch(tsdn, shard, &dalloc_list, deferred_work_generated);