From 38e42d311c1844a66e8ced84551621de41e42b85 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 10 Mar 2015 18:15:40 -0700 Subject: [PATCH] Refactor dirty run linkage to reduce sizeof(extent_node_t). --- include/jemalloc/internal/arena.h | 50 ++++++++--- include/jemalloc/internal/extent.h | 12 +-- include/jemalloc/internal/private_symbols.txt | 1 + src/arena.c | 89 ++++++++++--------- 4 files changed, 95 insertions(+), 57 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 50b296e4..de298e55 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -23,6 +23,7 @@ */ #define LG_DIRTY_MULT_DEFAULT 3 +typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; typedef struct arena_run_s arena_run_t; typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; @@ -120,6 +121,10 @@ struct arena_chunk_map_bits_s { #define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED }; +struct arena_runs_dirty_link_s { + qr(arena_runs_dirty_link_t) rd_link; +}; + /* * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just * like arena_chunk_map_bits_t. Two separate arrays are stored within each @@ -131,13 +136,13 @@ struct arena_chunk_map_misc_s { * * 1) arena_t's runs_avail tree. * 2) arena_run_t conceptually uses this linkage for in-use non-full - * runs, rather than directly embedding linkage. + * runs, rather than directly embedding linkage. */ rb_node(arena_chunk_map_misc_t) rb_link; union { /* Linkage for list of dirty runs. */ - qr(arena_chunk_map_misc_t) rd_link; + arena_runs_dirty_link_t rd; /* Profile counters, used for large object runs. */ prof_tctx_t *prof_tctx; @@ -324,15 +329,27 @@ struct arena_s { * * LRU-----------------------------------------------------------MRU * - * ______________ ___ ___ - * ...-->|chunks_cache|<--------->|c|<-------------------->|c|<--... - * -------------- |h| |h| - * ____________ _____ |u| _____ _____ |u| - * ...-->|runs_dirty|<-->|run|<-->|n|<-->|run|<-->|run|<-->|n|<--... - * ------------ ----- |k| ----- ----- |k| - * --- --- + * /------------------\ + * | arena | + * | | + * | /------------\ | /-----------\ + * ...---->|chunks_cache|<---------------------->| chunk |<--... + * | \------------/ | | | + * | | | | + * | | /---\ /---\ | | + * | | |run| |run| | | + * | | | | | | | | + * | /----------\ | |---| |---| | /-----\ | + * ...----->|runs_dirty|<---->|rd |<---->|rd |<---->|rdelm|<-----... + * | \----------/ | |---| |---| | \-----/ | + * | | | | | | | | + * | | | | | | | | + * | | \---/ \---/ | | + * | | | | + * | | | | + * \------------------/ \-----------/ */ - arena_chunk_map_misc_t runs_dirty; + arena_runs_dirty_link_t runs_dirty; extent_node_t chunks_cache; /* Extant huge allocations. */ @@ -465,6 +482,7 @@ arena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, size_t pageind); size_t arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm); void *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); +arena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd); arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbitsp_read(size_t *mapbitsp); @@ -556,6 +574,18 @@ arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm) return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); } +JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * +arena_rd_to_miscelm(arena_runs_dirty_link_t *rd) +{ + arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t + *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); + + assert(arena_miscelm_to_pageind(miscelm) >= map_bias); + assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); + + return (miscelm); +} + JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * arena_run_to_miscelm(arena_run_t *run) { diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 81ff40b8..5dbc04a5 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -34,7 +34,7 @@ struct extent_node_s { prof_tctx_t *en_prof_tctx; /* Linkage for arena's runs_dirty and chunks_cache rings. */ - arena_chunk_map_misc_t runs_dirty; + arena_runs_dirty_link_t rdelm; qr(extent_node_t) cc_link; union { @@ -79,7 +79,7 @@ void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, bool zeroed); void extent_node_dirty_linkage_init(extent_node_t *node); void extent_node_dirty_insert(extent_node_t *node, - arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty); + arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty); void extent_node_dirty_remove(extent_node_t *node); #endif @@ -186,16 +186,16 @@ JEMALLOC_INLINE void extent_node_dirty_linkage_init(extent_node_t *node) { - qr_new(&node->runs_dirty, rd_link); + qr_new(&node->rdelm, rd_link); qr_new(node, cc_link); } JEMALLOC_INLINE void extent_node_dirty_insert(extent_node_t *node, - arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty) + arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty) { - qr_meld(runs_dirty, &node->runs_dirty, rd_link); + qr_meld(runs_dirty, &node->rdelm, rd_link); qr_meld(chunks_dirty, node, cc_link); } @@ -203,7 +203,7 @@ JEMALLOC_INLINE void extent_node_dirty_remove(extent_node_t *node) { - qr_remove(&node->runs_dirty, rd_link); + qr_remove(&node->rdelm, rd_link); qr_remove(node, cc_link); } diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 7c217c74..d086db18 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -81,6 +81,7 @@ arena_quarantine_junk_small arena_ralloc arena_ralloc_junk_large arena_ralloc_no_move +arena_rd_to_miscelm arena_redzone_corruption arena_run_regind arena_run_to_miscelm diff --git a/src/arena.c b/src/arena.c index 6f4197b2..5d792f99 100644 --- a/src/arena.c +++ b/src/arena.c @@ -136,8 +136,8 @@ arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - qr_new(miscelm, rd_link); - qr_meld(&arena->runs_dirty, miscelm, rd_link); + qr_new(&miscelm->rd, rd_link); + qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link); arena->ndirty += npages; } @@ -153,7 +153,7 @@ arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - qr_remove(miscelm, rd_link); + qr_remove(&miscelm->rd, rd_link); assert(arena->ndirty >= npages); arena->ndirty -= npages; } @@ -1056,22 +1056,23 @@ static size_t arena_dirty_count(arena_t *arena) { size_t ndirty = 0; - arena_chunk_map_misc_t *runselm; + arena_runs_dirty_link_t *rdelm; extent_node_t *chunkselm; - for (runselm = qr_next(&arena->runs_dirty, rd_link), + for (rdelm = qr_next(&arena->runs_dirty, rd_link), chunkselm = qr_next(&arena->chunks_cache, cc_link); - runselm != &arena->runs_dirty; runselm = qr_next(runselm, - rd_link)) { + rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { size_t npages; - if (runselm == &chunkselm->runs_dirty) { + if (rdelm == &chunkselm->rdelm) { npages = extent_node_size_get(chunkselm) >> LG_PAGE; chunkselm = qr_next(chunkselm, cc_link); } else { - arena_chunk_t *chunk = (arena_chunk_t - *)CHUNK_ADDR2BASE(runselm); - size_t pageind = arena_miscelm_to_pageind(runselm); + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( + rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); assert(arena_mapbits_allocated_get(chunk, pageind) == 0); assert(arena_mapbits_large_get(chunk, pageind) == 0); @@ -1107,21 +1108,21 @@ arena_compute_npurge(arena_t *arena, bool all) static size_t arena_stash_dirty(arena_t *arena, bool all, size_t npurge, - arena_chunk_map_misc_t *purge_runs_sentinel, + arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { - arena_chunk_map_misc_t *runselm, *runselm_next; + arena_runs_dirty_link_t *rdelm, *rdelm_next; extent_node_t *chunkselm; size_t nstashed = 0; /* Stash at least npurge pages. */ - for (runselm = qr_next(&arena->runs_dirty, rd_link), + for (rdelm = qr_next(&arena->runs_dirty, rd_link), chunkselm = qr_next(&arena->chunks_cache, cc_link); - runselm != &arena->runs_dirty; runselm = runselm_next) { + rdelm != &arena->runs_dirty; rdelm = rdelm_next) { size_t npages; - runselm_next = qr_next(runselm, rd_link); + rdelm_next = qr_next(rdelm, rd_link); - if (runselm == &chunkselm->runs_dirty) { + if (rdelm == &chunkselm->rdelm) { extent_node_t *chunkselm_next; bool zero; UNUSED void *chunk; @@ -1144,9 +1145,11 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, chunkselm = chunkselm_next; } else { arena_chunk_t *chunk = - (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); - size_t pageind = arena_miscelm_to_pageind(runselm); - arena_run_t *run = &runselm->run; + (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); + arena_run_t *run = &miscelm->run; size_t run_size = arena_mapbits_unallocated_size_get(chunk, pageind); @@ -1167,12 +1170,12 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, arena_run_split_large(arena, run, run_size, false); /* Stash. */ if (false) - qr_new(runselm, rd_link); /* Redundant. */ + qr_new(rdelm, rd_link); /* Redundant. */ else { - assert(qr_next(runselm, rd_link) == runselm); - assert(qr_prev(runselm, rd_link) == runselm); + assert(qr_next(rdelm, rd_link) == rdelm); + assert(qr_prev(rdelm, rd_link) == rdelm); } - qr_meld(purge_runs_sentinel, runselm, rd_link); + qr_meld(purge_runs_sentinel, rdelm, rd_link); } nstashed += npages; @@ -1184,11 +1187,12 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, } static size_t -arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, +arena_purge_stashed(arena_t *arena, + arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { size_t npurged, nmadvise; - arena_chunk_map_misc_t *runselm; + arena_runs_dirty_link_t *rdelm; extent_node_t *chunkselm; if (config_stats) @@ -1196,13 +1200,12 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, npurged = 0; malloc_mutex_unlock(&arena->lock); - for (runselm = qr_next(purge_runs_sentinel, rd_link), + for (rdelm = qr_next(purge_runs_sentinel, rd_link), chunkselm = qr_next(purge_chunks_sentinel, cc_link); - runselm != purge_runs_sentinel; runselm = qr_next(runselm, - rd_link)) { + rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { size_t npages; - if (runselm == &chunkselm->runs_dirty) { + if (rdelm == &chunkselm->rdelm) { size_t size = extent_node_size_get(chunkselm); bool unzeroed; @@ -1216,8 +1219,10 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, size_t pageind, run_size, flag_unzeroed, i; bool unzeroed; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); - pageind = arena_miscelm_to_pageind(runselm); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + pageind = arena_miscelm_to_pageind(miscelm); run_size = arena_mapbits_large_size_get(chunk, pageind); npages = run_size >> LG_PAGE; @@ -1259,18 +1264,18 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, static void arena_unstash_purged(arena_t *arena, - arena_chunk_map_misc_t *purge_runs_sentinel, + arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { - arena_chunk_map_misc_t *runselm, *runselm_next; + arena_runs_dirty_link_t *rdelm, *rdelm_next; extent_node_t *chunkselm; /* Deallocate runs. */ - for (runselm = qr_next(purge_runs_sentinel, rd_link), + for (rdelm = qr_next(purge_runs_sentinel, rd_link), chunkselm = qr_next(purge_chunks_sentinel, cc_link); - runselm != purge_runs_sentinel; runselm = runselm_next) { - runselm_next = qr_next(runselm, rd_link); - if (runselm == &chunkselm->runs_dirty) { + rdelm != purge_runs_sentinel; rdelm = rdelm_next) { + rdelm_next = qr_next(rdelm, rd_link); + if (rdelm == &chunkselm->rdelm) { extent_node_t *chunkselm_next = qr_next(chunkselm, cc_link); void *addr = extent_node_addr_get(chunkselm); @@ -1281,8 +1286,10 @@ arena_unstash_purged(arena_t *arena, chunkselm = chunkselm_next; chunk_dalloc_arena(arena, addr, size, zeroed); } else { - arena_run_t *run = &runselm->run; - qr_remove(runselm, rd_link); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + arena_run_t *run = &miscelm->run; + qr_remove(rdelm, rd_link); arena_run_dalloc(arena, run, false, true); } } @@ -1292,7 +1299,7 @@ void arena_purge(arena_t *arena, bool all) { size_t npurge, npurgeable, npurged; - arena_chunk_map_misc_t purge_runs_sentinel; + arena_runs_dirty_link_t purge_runs_sentinel; extent_node_t purge_chunks_sentinel; /*