From be8e59f5a64ef775c9694aee0d6a87d92336d303 Mon Sep 17 00:00:00 2001 From: Ben Maurer Date: Sat, 5 Apr 2014 15:59:08 -0700 Subject: [PATCH 001/352] Don't dereference chunk->arena in free() hot path When you call free() we load chunk->arena even though that data isn't used on the tcache hot path. In profiling some FB applications, I found that ~30% of the dTLB misses in the free() function come from this line. With 4 MB chunks, the arena_chunk_t->map is ~ 32 KB (1024 pages in the chunk, 4 8 byte pointers in arena_chunk_map_t). This means there's only a 1/8 chance of the page containing chunk->arena also comtaining the map bits. --- include/jemalloc/internal/arena.h | 11 ++++------- include/jemalloc/internal/jemalloc_internal.h.in | 2 +- src/jemalloc.c | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 9d000c03..b899888d 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -495,8 +495,7 @@ prof_ctx_t *arena_prof_ctx_get(const void *ptr); void arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx); void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); size_t arena_salloc(const void *ptr, bool demote); -void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, - bool try_tcache); +void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) @@ -1022,13 +1021,11 @@ arena_salloc(const void *ptr, bool demote) } JEMALLOC_ALWAYS_INLINE void -arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) +arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) { size_t pageind, mapbits; tcache_t *tcache; - assert(arena != NULL); - assert(chunk->arena == arena); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); @@ -1043,7 +1040,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) binind = arena_ptr_small_binind_get(ptr, mapbits); tcache_dalloc_small(tcache, ptr, binind); } else - arena_dalloc_small(arena, chunk, ptr, pageind); + arena_dalloc_small(chunk->arena, chunk, ptr, pageind); } else { size_t size = arena_mapbits_large_size_get(chunk, pageind); @@ -1053,7 +1050,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) tcache_get(false)) != NULL) { tcache_dalloc_large(tcache, ptr, size); } else - arena_dalloc_large(arena, chunk, ptr); + arena_dalloc_large(chunk->arena, chunk, ptr); } } # endif /* JEMALLOC_ARENA_INLINE_B */ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 574bbb14..9c79ae00 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -895,7 +895,7 @@ idalloct(void *ptr, bool try_tcache) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(chunk->arena, chunk, ptr, try_tcache); + arena_dalloc(chunk, ptr, try_tcache); else huge_dalloc(ptr, true); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 204778bc..558dbb20 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2103,7 +2103,7 @@ a0free(void *ptr) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(chunk->arena, chunk, ptr, false); + arena_dalloc(chunk, ptr, false); else huge_dalloc(ptr, true); } From f9ff60346d7c25ad653ea062e496a5d0864233b2 Mon Sep 17 00:00:00 2001 From: Ben Maurer Date: Sun, 6 Apr 2014 13:24:16 -0700 Subject: [PATCH 002/352] refactoring for bits splitting --- src/arena.c | 76 ++++++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/src/arena.c b/src/arena.c index dad707b6..3cb62601 100644 --- a/src/arena.c +++ b/src/arena.c @@ -53,6 +53,22 @@ static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, /******************************************************************************/ +JEMALLOC_INLINE_C size_t +arena_mapelm_to_pageind(arena_chunk_map_t *mapelm) +{ + uintptr_t map_offset = + CHUNK_ADDR2OFFSET(mapelm) - offsetof(arena_chunk_t, map); + + return ((map_offset / sizeof(arena_chunk_map_t)) + map_bias); +} + +JEMALLOC_INLINE_C size_t +arena_mapelm_to_bits(arena_chunk_map_t *mapelm) +{ + + return (mapelm->bits); +} + static inline int arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) { @@ -73,26 +89,19 @@ static inline int arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) { int ret; - size_t a_size = a->bits & ~PAGE_MASK; - size_t b_size = b->bits & ~PAGE_MASK; + size_t a_size; + size_t b_size = arena_mapelm_to_bits(b) & ~PAGE_MASK; + uintptr_t a_mapelm = (uintptr_t)a; + uintptr_t b_mapelm = (uintptr_t)b; + + if (a_mapelm & CHUNK_MAP_KEY) + a_size = a_mapelm & ~PAGE_MASK; + else + a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; ret = (a_size > b_size) - (a_size < b_size); - if (ret == 0) { - uintptr_t a_mapelm, b_mapelm; - - if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) - a_mapelm = (uintptr_t)a; - else { - /* - * Treat keys as though they are lower than anything - * else. - */ - a_mapelm = 0; - } - b_mapelm = (uintptr_t)b; - + if (ret == 0 && (!(a_mapelm & CHUNK_MAP_KEY))) ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); - } return (ret); } @@ -663,15 +672,14 @@ static arena_run_t * arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { arena_run_t *run; - arena_chunk_map_t *mapelm, key; + arena_chunk_map_t *mapelm; + arena_chunk_map_t *key; - key.bits = size | CHUNK_MAP_KEY; - mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); + key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); + mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (mapelm != NULL) { arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; + size_t pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << LG_PAGE)); @@ -718,15 +726,14 @@ static arena_run_t * arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) { arena_run_t *run; - arena_chunk_map_t *mapelm, key; + arena_chunk_map_t *mapelm; + arena_chunk_map_t *key; - key.bits = size | CHUNK_MAP_KEY; - mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); + key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); + mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (mapelm != NULL) { arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; + size_t pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << LG_PAGE)); @@ -897,8 +904,7 @@ arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, bool unzeroed; size_t flag_unzeroed, i; - pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t)) + map_bias; + pageind = arena_mapelm_to_pageind(mapelm); npages = arena_mapbits_large_size_get(chunk, pageind) >> LG_PAGE; assert(pageind + npages <= chunk_npages); @@ -942,8 +948,7 @@ arena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, mapelm = ql_first(mapelms)) { arena_run_t *run; - pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t)) + map_bias; + pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << LG_PAGE)); ql_remove(mapelms, mapelm, u.ql_link); @@ -1307,8 +1312,7 @@ arena_bin_runs_first(arena_bin_t *bin) arena_run_t *run; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t))) + map_bias; + pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); @@ -1882,7 +1886,7 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); bin = run->bin; - binind = arena_ptr_small_binind_get(ptr, mapelm->bits); + binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); bin_info = &arena_bin_info[binind]; if (config_fill || config_stats) size = bin_info->reg_size; From 9b0cbf0850b130a9b0a8c58bd10b2926b2083510 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 11 Apr 2014 14:24:51 -0700 Subject: [PATCH 003/352] Remove support for non-prof-promote heap profiling metadata. Make promotion of sampled small objects to large objects mandatory, so that profiling metadata can always be stored in the chunk map, rather than requiring one pointer per small region in each small-region page run. In practice the non-prof-promote code was only useful when using jemalloc to track all objects and report them as leaks at program exit. However, Valgrind is at least as good a tool for this particular use case. Furthermore, the non-prof-promote code is getting in the way of some optimizations that will make heap profiling much cheaper for the predominant use case (sampling a small representative proportion of all allocations). --- include/jemalloc/internal/arena.h | 66 +++---------------- include/jemalloc/internal/private_symbols.txt | 1 - include/jemalloc/internal/prof.h | 20 ++---- include/jemalloc/internal/size_classes.sh | 5 +- include/jemalloc/internal/tcache.h | 2 +- src/arena.c | 21 ------ src/jemalloc.c | 16 ++--- src/prof.c | 7 +- 8 files changed, 28 insertions(+), 110 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index b899888d..0e14c2c4 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -110,7 +110,6 @@ struct arena_chunk_map_s { * p : run page offset * s : run size * n : binind for size class; large objects set these to BININD_INVALID - * except for promoted allocations (see prof_promote) * x : don't care * - : 0 * + : 1 @@ -216,8 +215,6 @@ struct arena_run_s { * | ... | * bitmap_offset | bitmap | * | ... | - * ctx0_offset | ctx map | - * | ... | * |--------------------| * | redzone | * reg0_offset | region 0 | @@ -270,12 +267,6 @@ struct arena_bin_info_s { */ bitmap_info_t bitmap_info; - /* - * Offset of first (prof_ctx_t *) in a run header for this bin's size - * class, or 0 if (config_prof == false || opt_prof == false). - */ - uint32_t ctx0_offset; - /* Offset of first region in a run for this bin's size class. */ uint32_t reg0_offset; }; @@ -492,7 +483,7 @@ size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); prof_ctx_t *arena_prof_ctx_get(const void *ptr); -void arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx); +void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); @@ -879,31 +870,16 @@ arena_prof_ctx_get(const void *ptr) pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - if (prof_promote) - ret = (prof_ctx_t *)(uintptr_t)1U; - else { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << - LG_PAGE)); - size_t binind = arena_ptr_small_binind_get(ptr, - mapbits); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - unsigned regind; - - regind = arena_run_regind(run, bin_info, ptr); - ret = *(prof_ctx_t **)((uintptr_t)run + - bin_info->ctx0_offset + (regind * - sizeof(prof_ctx_t *))); - } - } else + if ((mapbits & CHUNK_MAP_LARGE) == 0) + ret = (prof_ctx_t *)(uintptr_t)1U; + else ret = arena_mapp_get(chunk, pageind)->prof_ctx; return (ret); } JEMALLOC_INLINE void -arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) +arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) { arena_chunk_t *chunk; size_t pageind; @@ -916,31 +892,8 @@ arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (usize > SMALL_MAXCLASS || (prof_promote && - ((uintptr_t)ctx != (uintptr_t)1U || arena_mapbits_large_get(chunk, - pageind) != 0))) { - assert(arena_mapbits_large_get(chunk, pageind) != 0); + if (arena_mapbits_large_get(chunk, pageind) != 0) arena_mapp_get(chunk, pageind)->prof_ctx = ctx; - } else { - assert(arena_mapbits_large_get(chunk, pageind) == 0); - if (prof_promote == false) { - size_t mapbits = arena_mapbits_get(chunk, pageind); - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << - LG_PAGE)); - size_t binind; - arena_bin_info_t *bin_info; - unsigned regind; - - binind = arena_ptr_small_binind_get(ptr, mapbits); - bin_info = &arena_bin_info[binind]; - regind = arena_run_regind(run, bin_info, ptr); - - *((prof_ctx_t **)((uintptr_t)run + - bin_info->ctx0_offset + (regind * sizeof(prof_ctx_t - *)))) = ctx; - } - } } JEMALLOC_ALWAYS_INLINE void * @@ -989,7 +942,7 @@ arena_salloc(const void *ptr, bool demote) assert(arena_mapbits_allocated_get(chunk, pageind) != 0); binind = arena_mapbits_binind_get(chunk, pageind); if (binind == BININD_INVALID || (config_prof && demote == false && - prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) { + arena_mapbits_large_get(chunk, pageind) != 0)) { /* * Large allocation. In the common case (demote == true), and * as this is an inline function, most callers will only end up @@ -1007,10 +960,7 @@ arena_salloc(const void *ptr, bool demote) assert(arena_mapbits_dirty_get(chunk, pageind) == arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); } else { - /* - * Small allocation (possibly promoted to a large object due to - * prof_promote). - */ + /* Small allocation (possibly promoted to a large object). */ assert(arena_mapbits_large_get(chunk, pageind) != 0 || arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)) == binind); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 93516d24..f52d49f9 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -303,7 +303,6 @@ prof_mdump prof_postfork_child prof_postfork_parent prof_prefork -prof_promote prof_realloc prof_sample_accum_update prof_sample_threshold_update diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 6f162d21..56014f18 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -220,12 +220,6 @@ extern char opt_prof_prefix[ */ extern uint64_t prof_interval; -/* - * If true, promote small sampled objects to large objects, since small run - * headers do not have embedded profile context pointers. - */ -extern bool prof_promote; - void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt, unsigned nignore); prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); @@ -308,7 +302,7 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) prof_tdata_t *prof_tdata_get(bool create); void prof_sample_threshold_update(prof_tdata_t *prof_tdata); prof_ctx_t *prof_ctx_get(const void *ptr); -void prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx); +void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); bool prof_sample_accum_update(size_t size); void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, @@ -405,7 +399,7 @@ prof_ctx_get(const void *ptr) } JEMALLOC_INLINE void -prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) +prof_ctx_set(const void *ptr, prof_ctx_t *ctx) { arena_chunk_t *chunk; @@ -415,7 +409,7 @@ prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - arena_prof_ctx_set(ptr, usize, ctx); + arena_prof_ctx_set(ptr, ctx); } else huge_prof_ctx_set(ptr, ctx); } @@ -471,7 +465,7 @@ prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) } if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, usize, cnt->ctx); + prof_ctx_set(ptr, cnt->ctx); cnt->epoch++; /*********/ @@ -491,7 +485,7 @@ prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) mb_write(); /*********/ } else - prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U); + prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); } JEMALLOC_INLINE void @@ -539,10 +533,10 @@ prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, if ((uintptr_t)told_cnt > (uintptr_t)1U) told_cnt->epoch++; if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, usize, cnt->ctx); + prof_ctx_set(ptr, cnt->ctx); cnt->epoch++; } else if (ptr != NULL) - prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U); + prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); /*********/ mb_write(); /*********/ diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 29c80c1f..960674aa 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -94,9 +94,8 @@ cat < 255) # error "Too many small size classes" diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index c3d4b58d..51974136 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -354,7 +354,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) if (ret == NULL) return (NULL); } else { - if (config_prof && prof_promote && size == PAGE) { + if (config_prof && size == PAGE) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> diff --git a/src/arena.c b/src/arena.c index 3cb62601..d5741000 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2373,7 +2373,6 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) uint32_t try_nregs, good_nregs; uint32_t try_hdr_size, good_hdr_size; uint32_t try_bitmap_offset, good_bitmap_offset; - uint32_t try_ctx0_offset, good_ctx0_offset; uint32_t try_redzone0_offset, good_redzone0_offset; assert(min_run_size >= PAGE); @@ -2428,14 +2427,6 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); - if (config_prof && opt_prof && prof_promote == false) { - /* Pad to a quantum boundary. */ - try_hdr_size = QUANTUM_CEILING(try_hdr_size); - try_ctx0_offset = try_hdr_size; - /* Add space for one (prof_ctx_t *) per region. */ - try_hdr_size += try_nregs * sizeof(prof_ctx_t *); - } else - try_ctx0_offset = 0; try_redzone0_offset = try_run_size - (try_nregs * bin_info->reg_interval) - pad_size; } while (try_hdr_size > try_redzone0_offset); @@ -2449,7 +2440,6 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) good_nregs = try_nregs; good_hdr_size = try_hdr_size; good_bitmap_offset = try_bitmap_offset; - good_ctx0_offset = try_ctx0_offset; good_redzone0_offset = try_redzone0_offset; /* Try more aggressive settings. */ @@ -2469,16 +2459,6 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); - if (config_prof && opt_prof && prof_promote == false) { - /* Pad to a quantum boundary. */ - try_hdr_size = QUANTUM_CEILING(try_hdr_size); - try_ctx0_offset = try_hdr_size; - /* - * Add space for one (prof_ctx_t *) per region. - */ - try_hdr_size += try_nregs * - sizeof(prof_ctx_t *); - } try_redzone0_offset = try_run_size - (try_nregs * bin_info->reg_interval) - pad_size; } while (try_hdr_size > try_redzone0_offset); @@ -2494,7 +2474,6 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) bin_info->run_size = good_run_size; bin_info->nregs = good_nregs; bin_info->bitmap_offset = good_bitmap_offset; - bin_info->ctx0_offset = good_ctx0_offset; bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs diff --git a/src/jemalloc.c b/src/jemalloc.c index 558dbb20..816a12e0 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -853,7 +853,7 @@ imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) if (cnt == NULL) return (NULL); - if (prof_promote && usize <= SMALL_MAXCLASS) { + if (usize <= SMALL_MAXCLASS) { p = imalloc(SMALL_MAXCLASS+1); if (p == NULL) return (NULL); @@ -952,7 +952,7 @@ imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) if (cnt == NULL) return (NULL); - if (prof_promote && usize <= SMALL_MAXCLASS) { + if (usize <= SMALL_MAXCLASS) { assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0); p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment, false); @@ -1086,7 +1086,7 @@ icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) if (cnt == NULL) return (NULL); - if (prof_promote && usize <= SMALL_MAXCLASS) { + if (usize <= SMALL_MAXCLASS) { p = icalloc(SMALL_MAXCLASS+1); if (p == NULL) return (NULL); @@ -1183,7 +1183,7 @@ irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) if (cnt == NULL) return (NULL); - if (prof_promote && usize <= SMALL_MAXCLASS) { + if (usize <= SMALL_MAXCLASS) { p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false); if (p == NULL) return (NULL); @@ -1395,7 +1395,7 @@ imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, if (cnt == NULL) return (NULL); - if (prof_promote && usize <= SMALL_MAXCLASS) { + if (usize <= SMALL_MAXCLASS) { size_t usize_promoted = (alignment == 0) ? s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); assert(usize_promoted != 0); @@ -1492,7 +1492,7 @@ irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, if (cnt == NULL) return (NULL); - if (prof_promote && usize <= SMALL_MAXCLASS) { + if (usize <= SMALL_MAXCLASS) { p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); @@ -1639,8 +1639,8 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, if (cnt == NULL) return (old_usize); /* Use minimum usize to determine whether promotion may happen. */ - if (prof_promote && ((alignment == 0) ? s2u(size) : sa2u(size, - alignment)) <= SMALL_MAXCLASS) { + if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= + SMALL_MAXCLASS) { if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), alignment, zero)) diff --git a/src/prof.c b/src/prof.c index 7722b7b4..1b1f7a84 100644 --- a/src/prof.c +++ b/src/prof.c @@ -32,7 +32,6 @@ char opt_prof_prefix[ 1]; uint64_t prof_interval = 0; -bool prof_promote; /* * Table of mutexes that are shared among ctx's. These are leaf locks, so @@ -1300,8 +1299,8 @@ prof_boot1(void) cassert(config_prof); /* - * opt_prof and prof_promote must be in their final state before any - * arenas are initialized, so this function must be executed early. + * opt_prof must be in its final state before any arenas are + * initialized, so this function must be executed early. */ if (opt_prof_leak && opt_prof == false) { @@ -1317,8 +1316,6 @@ prof_boot1(void) opt_lg_prof_interval); } } - - prof_promote = (opt_prof && opt_lg_prof_sample > LG_PAGE); } bool From 9790b9667fd975b1f9a4f108f9d0a20ab265c6b6 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Apr 2014 22:32:31 -0700 Subject: [PATCH 004/352] Remove the *allocm() API, which is superceded by the *allocx() API. --- INSTALL | 3 - Makefile.in | 7 +- configure.ac | 21 +- doc/jemalloc.xml.in | 191 +----------------- .../jemalloc/internal/jemalloc_internal.h.in | 1 - include/jemalloc/internal/private_symbols.txt | 1 - include/jemalloc/jemalloc_defs.h.in | 3 - include/jemalloc/jemalloc_macros.h.in | 17 -- include/jemalloc/jemalloc_protos.h.in | 12 -- src/jemalloc.c | 85 -------- test/integration/MALLOCX_ARENA.c | 4 +- test/integration/allocm.c | 107 ---------- test/integration/rallocm.c | 111 ---------- test/unit/mq.c | 2 +- 14 files changed, 7 insertions(+), 558 deletions(-) delete mode 100644 test/integration/allocm.c delete mode 100644 test/integration/rallocm.c diff --git a/INSTALL b/INSTALL index 841704d2..8530643d 100644 --- a/INSTALL +++ b/INSTALL @@ -157,9 +157,6 @@ any of the following arguments (not a definitive list) to 'configure': --disable-valgrind Disable support for Valgrind. ---disable-experimental - Disable support for the experimental API (*allocm()). - --disable-zone-allocator Disable zone allocator for Darwin. This means jemalloc won't be hooked as the default allocator on OSX/iOS. diff --git a/Makefile.in b/Makefile.in index d6b7d6ea..f7aa7d87 100644 --- a/Makefile.in +++ b/Makefile.in @@ -48,7 +48,6 @@ cfgoutputs_in := @cfgoutputs_in@ cfgoutputs_out := @cfgoutputs_out@ enable_autogen := @enable_autogen@ enable_code_coverage := @enable_code_coverage@ -enable_experimental := @enable_experimental@ enable_zone_allocator := @enable_zone_allocator@ DSO_LDFLAGS = @DSO_LDFLAGS@ SOREV = @SOREV@ @@ -133,17 +132,13 @@ TESTS_UNIT_AUX := $(srcroot)test/unit/prof_accum_a.c \ TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/allocated.c \ $(srcroot)test/integration/mallocx.c \ + $(srcroot)test/integration/MALLOCX_ARENA.c \ $(srcroot)test/integration/mremap.c \ $(srcroot)test/integration/posix_memalign.c \ $(srcroot)test/integration/rallocx.c \ $(srcroot)test/integration/thread_arena.c \ $(srcroot)test/integration/thread_tcache_enabled.c \ $(srcroot)test/integration/xallocx.c -ifeq ($(enable_experimental), 1) -TESTS_INTEGRATION += $(srcroot)test/integration/allocm.c \ - $(srcroot)test/integration/MALLOCX_ARENA.c \ - $(srcroot)test/integration/rallocm.c -endif TESTS_STRESS := TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS) diff --git a/configure.ac b/configure.ac index 4de81dc1..04cefe95 100644 --- a/configure.ac +++ b/configure.ac @@ -44,7 +44,7 @@ AC_CACHE_CHECK([whether $1 is compilable], dnl ============================================================================ dnl Library revision. -rev=1 +rev=2 AC_SUBST([rev]) srcroot=$srcdir @@ -438,24 +438,6 @@ AC_CHECK_FUNC([valloc], [AC_DEFINE([JEMALLOC_OVERRIDE_VALLOC], [ ]) public_syms="${public_syms} valloc"]) -dnl Support the experimental API by default. -AC_ARG_ENABLE([experimental], - [AS_HELP_STRING([--disable-experimental], - [Disable support for the experimental API])], -[if test "x$enable_experimental" = "xno" ; then - enable_experimental="0" -else - enable_experimental="1" -fi -], -[enable_experimental="1"] -) -if test "x$enable_experimental" = "x1" ; then - AC_DEFINE([JEMALLOC_EXPERIMENTAL], [ ]) - public_syms="${public_syms} allocm dallocm nallocm rallocm sallocm" -fi -AC_SUBST([enable_experimental]) - dnl Do not compute test code coverage by default. GCOV_FLAGS= AC_ARG_ENABLE([code-coverage], @@ -1465,7 +1447,6 @@ AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([autogen : ${enable_autogen}]) -AC_MSG_RESULT([experimental : ${enable_experimental}]) AC_MSG_RESULT([cc-silence : ${enable_cc_silence}]) AC_MSG_RESULT([debug : ${enable_debug}]) AC_MSG_RESULT([code-coverage : ${enable_code_coverage}]) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index d8e2e711..a4232409 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -44,11 +44,6 @@ mallctlbymib malloc_stats_print malloc_usable_size - allocm - rallocm - sallocm - dallocm - nallocm --> general purpose memory allocation functions @@ -172,41 +167,6 @@ const char *malloc_conf; - - Experimental API - - int allocm - void **ptr - size_t *rsize - size_t size - int flags - - - int rallocm - void **ptr - size_t *rsize - size_t size - size_t extra - int flags - - - int sallocm - const void *ptr - size_t *rsize - int flags - - - int dallocm - void *ptr - int flags - - - int nallocm - size_t *rsize - size_t size - int flags - - @@ -449,116 +409,6 @@ for (i = 0; i < nbins; i++) { depended on, since such behavior is entirely implementation-dependent. - - Experimental API - The experimental API is subject to change or removal without regard - for backward compatibility. If - is specified during configuration, the experimental API is - omitted. - - The allocm, - rallocm, - sallocm, - dallocm, and - nallocm functions all have a - flags argument that can be used to specify - options. The functions only check the options that are contextually - relevant. Use bitwise or (|) operations to - specify one or more of the following: - - - ALLOCM_LG_ALIGN(la) - - - Align the memory allocation to start at an address - that is a multiple of (1 << - la). This macro does not validate - that la is within the valid - range. - - - ALLOCM_ALIGN(a) - - - Align the memory allocation to start at an address - that is a multiple of a, where - a is a power of two. This macro does not - validate that a is a power of 2. - - - - ALLOCM_ZERO - - Initialize newly allocated memory to contain zero - bytes. In the growing reallocation case, the real size prior to - reallocation defines the boundary between untouched bytes and those - that are initialized to contain zero bytes. If this macro is - absent, newly allocated memory is uninitialized. - - - ALLOCM_NO_MOVE - - For reallocation, fail rather than moving the - object. This constraint can apply to both growth and - shrinkage. - - - ALLOCM_ARENA(a) - - - Use the arena specified by the index - a (and by necessity bypass the thread - cache). This macro has no effect for huge regions, nor for regions - that were allocated via an arena other than the one specified. - This macro does not validate that a - specifies an arena index in the valid range. - - - - - The allocm function allocates at - least size bytes of memory, sets - *ptr to the base address of the allocation, and - sets *rsize to the real size of the allocation if - rsize is not NULL. Behavior - is undefined if size is 0, or - if request size overflows due to size class and/or alignment - constraints. - - The rallocm function resizes the - allocation at *ptr to be at least - size bytes, sets *ptr to - the base address of the allocation if it moved, and sets - *rsize to the real size of the allocation if - rsize is not NULL. If - extra is non-zero, an attempt is made to resize - the allocation to be at least (size + - extra) bytes, though inability to allocate - the extra byte(s) will not by itself result in failure. Behavior is - undefined if size is 0, if - request size overflows due to size class and/or alignment constraints, or - if (size + - extra > - SIZE_T_MAX). - - The sallocm function sets - *rsize to the real size of the allocation. - - The dallocm function causes the - memory referenced by ptr to be made available for - future allocations. - - The nallocm function allocates no - memory, but it performs the same size computation as the - allocm function, and if - rsize is not NULL it sets - *rsize to the real size of the allocation that - would result from the equivalent allocm - function call. Behavior is undefined if size is - 0, or if request size overflows due to size class - and/or alignment constraints. - TUNING @@ -1076,9 +926,8 @@ for (i = 0; i < nbins; i++) { Zero filling enabled/disabled. If enabled, each byte of uninitialized allocated memory will be initialized to 0. Note that this initialization only happens once for each byte, so - realloc, - rallocx and - rallocm calls do not zero memory that + realloc and + rallocx calls do not zero memory that was previously allocated. This is intended for debugging and will impact performance negatively. This option is disabled by default. @@ -2253,42 +2102,6 @@ malloc_conf = "xmalloc:true";]]> returns the usable size of the allocation pointed to by ptr. - - Experimental API - The allocm, - rallocm, - sallocm, - dallocm, and - nallocm functions return - ALLOCM_SUCCESS on success; otherwise they return an - error value. The allocm, - rallocm, and - nallocm functions will fail if: - - - ALLOCM_ERR_OOM - - Out of memory. Insufficient contiguous memory was - available to service the allocation request. The - allocm function additionally sets - *ptr to NULL, whereas - the rallocm function leaves - *ptr unmodified. - - - The rallocm function will also - fail if: - - - ALLOCM_ERR_NOT_MOVED - - ALLOCM_NO_MOVE was specified, - but the reallocation request could not be serviced without moving - the object. - - - - ENVIRONMENT diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 9c79ae00..a374e2a3 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -230,7 +230,6 @@ static const bool config_ivsalloc = #include "jemalloc/internal/jemalloc_internal_macros.h" #define MALLOCX_LG_ALIGN_MASK ((int)0x3f) -#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) /* Smallest size class to support. */ #define LG_TINY_MIN 3 diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index f52d49f9..e1cb28f8 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -211,7 +211,6 @@ huge_prof_ctx_set huge_ralloc huge_ralloc_no_move huge_salloc -iallocm icalloc icalloct idalloc diff --git a/include/jemalloc/jemalloc_defs.h.in b/include/jemalloc/jemalloc_defs.h.in index eb38d710..ce6c6987 100644 --- a/include/jemalloc/jemalloc_defs.h.in +++ b/include/jemalloc/jemalloc_defs.h.in @@ -1,9 +1,6 @@ /* Defined if __attribute__((...)) syntax is supported. */ #undef JEMALLOC_HAVE_ATTR -/* Support the experimental API. */ -#undef JEMALLOC_EXPERIMENTAL - /* * Define overrides for non-standard allocator-related functions if they are * present on the system. diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in index 13dbdd91..1530f9ca 100644 --- a/include/jemalloc/jemalloc_macros.h.in +++ b/include/jemalloc/jemalloc_macros.h.in @@ -19,23 +19,6 @@ /* Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". */ # define MALLOCX_ARENA(a) ((int)(((a)+1) << 8)) -#ifdef JEMALLOC_EXPERIMENTAL -# define ALLOCM_LG_ALIGN(la) (la) -# if LG_SIZEOF_PTR == 2 -# define ALLOCM_ALIGN(a) (ffs(a)-1) -# else -# define ALLOCM_ALIGN(a) \ - ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) -# endif -# define ALLOCM_ZERO ((int)0x40) -# define ALLOCM_NO_MOVE ((int)0x80) -/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */ -# define ALLOCM_ARENA(a) ((int)(((a)+1) << 8)) -# define ALLOCM_SUCCESS 0 -# define ALLOCM_ERR_OOM 1 -# define ALLOCM_ERR_NOT_MOVED 2 -#endif - #ifdef JEMALLOC_HAVE_ATTR # define JEMALLOC_ATTR(s) __attribute__((s)) # define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 25446de3..59aeee11 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -44,15 +44,3 @@ JEMALLOC_EXPORT void * @je_@memalign(size_t alignment, size_t size) #ifdef JEMALLOC_OVERRIDE_VALLOC JEMALLOC_EXPORT void * @je_@valloc(size_t size) JEMALLOC_ATTR(malloc); #endif - -#ifdef JEMALLOC_EXPERIMENTAL -JEMALLOC_EXPORT int @je_@allocm(void **ptr, size_t *rsize, size_t size, - int flags) JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int @je_@rallocm(void **ptr, size_t *rsize, size_t size, - size_t extra, int flags) JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int @je_@sallocm(const void *ptr, size_t *rsize, int flags) - JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int @je_@dallocm(void *ptr, int flags) - JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int @je_@nallocm(size_t *rsize, size_t size, int flags); -#endif diff --git a/src/jemalloc.c b/src/jemalloc.c index 816a12e0..0de59408 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1867,91 +1867,6 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) * End non-standard functions. */ /******************************************************************************/ -/* - * Begin experimental functions. - */ -#ifdef JEMALLOC_EXPERIMENTAL - -int -je_allocm(void **ptr, size_t *rsize, size_t size, int flags) -{ - void *p; - - assert(ptr != NULL); - - p = je_mallocx(size, flags); - if (p == NULL) - return (ALLOCM_ERR_OOM); - if (rsize != NULL) - *rsize = isalloc(p, config_prof); - *ptr = p; - return (ALLOCM_SUCCESS); -} - -int -je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) -{ - int ret; - bool no_move = flags & ALLOCM_NO_MOVE; - - assert(ptr != NULL); - assert(*ptr != NULL); - assert(size != 0); - assert(SIZE_T_MAX - size >= extra); - - if (no_move) { - size_t usize = je_xallocx(*ptr, size, extra, flags); - ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; - if (rsize != NULL) - *rsize = usize; - } else { - void *p = je_rallocx(*ptr, size+extra, flags); - if (p != NULL) { - *ptr = p; - ret = ALLOCM_SUCCESS; - } else - ret = ALLOCM_ERR_OOM; - if (rsize != NULL) - *rsize = isalloc(*ptr, config_prof); - } - return (ret); -} - -int -je_sallocm(const void *ptr, size_t *rsize, int flags) -{ - - assert(rsize != NULL); - *rsize = je_sallocx(ptr, flags); - return (ALLOCM_SUCCESS); -} - -int -je_dallocm(void *ptr, int flags) -{ - - je_dallocx(ptr, flags); - return (ALLOCM_SUCCESS); -} - -int -je_nallocm(size_t *rsize, size_t size, int flags) -{ - size_t usize; - - usize = je_nallocx(size, flags); - if (usize == 0) - return (ALLOCM_ERR_OOM); - if (rsize != NULL) - *rsize = usize; - return (ALLOCM_SUCCESS); -} - -#endif -/* - * End experimental functions. - */ -/******************************************************************************/ /* * The following functions are used by threading libraries for protection of * malloc during fork(). diff --git a/test/integration/MALLOCX_ARENA.c b/test/integration/MALLOCX_ARENA.c index 71cf6f25..695a5b66 100644 --- a/test/integration/MALLOCX_ARENA.c +++ b/test/integration/MALLOCX_ARENA.c @@ -34,7 +34,7 @@ thd_start(void *arg) return (NULL); } -TEST_BEGIN(test_ALLOCM_ARENA) +TEST_BEGIN(test_MALLOCX_ARENA) { thd_t thds[NTHREADS]; unsigned i; @@ -54,5 +54,5 @@ main(void) { return (test( - test_ALLOCM_ARENA)); + test_MALLOCX_ARENA)); } diff --git a/test/integration/allocm.c b/test/integration/allocm.c deleted file mode 100644 index 7b4ea0c2..00000000 --- a/test/integration/allocm.c +++ /dev/null @@ -1,107 +0,0 @@ -#include "test/jemalloc_test.h" - -#define CHUNK 0x400000 -#define MAXALIGN (((size_t)1) << 25) -#define NITER 4 - -TEST_BEGIN(test_basic) -{ - size_t nsz, rsz, sz; - void *p; - - sz = 42; - nsz = 0; - assert_d_eq(nallocm(&nsz, sz, 0), ALLOCM_SUCCESS, - "Unexpected nallocm() error"); - rsz = 0; - assert_d_eq(allocm(&p, &rsz, sz, 0), ALLOCM_SUCCESS, - "Unexpected allocm() error"); - assert_zu_ge(rsz, sz, "Real size smaller than expected"); - assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch"); - assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, - "Unexpected dallocm() error"); - - assert_d_eq(allocm(&p, NULL, sz, 0), ALLOCM_SUCCESS, - "Unexpected allocm() error"); - assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, - "Unexpected dallocm() error"); - - nsz = 0; - assert_d_eq(nallocm(&nsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS, - "Unexpected nallocm() error"); - rsz = 0; - assert_d_eq(allocm(&p, &rsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS, - "Unexpected allocm() error"); - assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch"); - assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, - "Unexpected dallocm() error"); -} -TEST_END - -TEST_BEGIN(test_alignment_and_size) -{ - int r; - size_t nsz, rsz, sz, alignment, total; - unsigned i; - void *ps[NITER]; - - for (i = 0; i < NITER; i++) - ps[i] = NULL; - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - for (sz = 1; - sz < 3 * alignment && sz < (1U << 31); - sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - nsz = 0; - r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment) | - ALLOCM_ZERO); - assert_d_eq(r, ALLOCM_SUCCESS, - "nallocm() error for alignment=%zu, " - "size=%zu (%#zx): %d", - alignment, sz, sz, r); - rsz = 0; - r = allocm(&ps[i], &rsz, sz, - ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); - assert_d_eq(r, ALLOCM_SUCCESS, - "allocm() error for alignment=%zu, " - "size=%zu (%#zx): %d", - alignment, sz, sz, r); - assert_zu_ge(rsz, sz, - "Real size smaller than expected for " - "alignment=%zu, size=%zu", alignment, sz); - assert_zu_eq(nsz, rsz, - "nallocm()/allocm() rsize mismatch for " - "alignment=%zu, size=%zu", alignment, sz); - assert_ptr_null( - (void *)((uintptr_t)ps[i] & (alignment-1)), - "%p inadequately aligned for" - " alignment=%zu, size=%zu", ps[i], - alignment, sz); - sallocm(ps[i], &rsz, 0); - total += rsz; - if (total >= (MAXALIGN << 1)) - break; - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - dallocm(ps[i], 0); - ps[i] = NULL; - } - } - } - } -} -TEST_END - -int -main(void) -{ - - return (test( - test_basic, - test_alignment_and_size)); -} diff --git a/test/integration/rallocm.c b/test/integration/rallocm.c deleted file mode 100644 index 33c11bb7..00000000 --- a/test/integration/rallocm.c +++ /dev/null @@ -1,111 +0,0 @@ -#include "test/jemalloc_test.h" - -TEST_BEGIN(test_same_size) -{ - void *p, *q; - size_t sz, tsz; - - assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, - "Unexpected allocm() error"); - - q = p; - assert_d_eq(rallocm(&q, &tsz, sz, 0, ALLOCM_NO_MOVE), ALLOCM_SUCCESS, - "Unexpected rallocm() error"); - assert_ptr_eq(q, p, "Unexpected object move"); - assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); - - assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, - "Unexpected dallocm() error"); -} -TEST_END - -TEST_BEGIN(test_extra_no_move) -{ - void *p, *q; - size_t sz, tsz; - - assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, - "Unexpected allocm() error"); - - q = p; - assert_d_eq(rallocm(&q, &tsz, sz, sz-42, ALLOCM_NO_MOVE), - ALLOCM_SUCCESS, "Unexpected rallocm() error"); - assert_ptr_eq(q, p, "Unexpected object move"); - assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); - - assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, - "Unexpected dallocm() error"); -} -TEST_END - -TEST_BEGIN(test_no_move_fail) -{ - void *p, *q; - size_t sz, tsz; - - assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, - "Unexpected allocm() error"); - - q = p; - assert_d_eq(rallocm(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE), - ALLOCM_ERR_NOT_MOVED, "Unexpected rallocm() result"); - assert_ptr_eq(q, p, "Unexpected object move"); - assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); - - assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, - "Unexpected dallocm() error"); -} -TEST_END - -TEST_BEGIN(test_grow_and_shrink) -{ - void *p, *q; - size_t tsz; -#define NCYCLES 3 - unsigned i, j; -#define NSZS 2500 - size_t szs[NSZS]; -#define MAXSZ ZU(12 * 1024 * 1024) - - assert_d_eq(allocm(&p, &szs[0], 1, 0), ALLOCM_SUCCESS, - "Unexpected allocm() error"); - - for (i = 0; i < NCYCLES; i++) { - for (j = 1; j < NSZS && szs[j-1] < MAXSZ; j++) { - q = p; - assert_d_eq(rallocm(&q, &szs[j], szs[j-1]+1, 0, 0), - ALLOCM_SUCCESS, - "Unexpected rallocm() error for size=%zu-->%zu", - szs[j-1], szs[j-1]+1); - assert_zu_ne(szs[j], szs[j-1]+1, - "Expected size to at least: %zu", szs[j-1]+1); - p = q; - } - - for (j--; j > 0; j--) { - q = p; - assert_d_eq(rallocm(&q, &tsz, szs[j-1], 0, 0), - ALLOCM_SUCCESS, - "Unexpected rallocm() error for size=%zu-->%zu", - szs[j], szs[j-1]); - assert_zu_eq(tsz, szs[j-1], - "Expected size=%zu, got size=%zu", szs[j-1], tsz); - p = q; - } - } - - assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, - "Unexpected dallocm() error"); -} -TEST_END - -int -main(void) -{ - - return (test( - test_same_size, - test_extra_no_move, - test_no_move_fail, - test_grow_and_shrink)); -} diff --git a/test/unit/mq.c b/test/unit/mq.c index f57e96af..bd289c54 100644 --- a/test/unit/mq.c +++ b/test/unit/mq.c @@ -54,7 +54,7 @@ thd_sender_start(void *arg) mq_msg_t *msg; void *p; p = mallocx(sizeof(mq_msg_t), 0); - assert_ptr_not_null(p, "Unexpected allocm() failure"); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); msg = (mq_msg_t *)p; mq_put(mq, msg); } From 24a4ba77e1e4c73488c6ac87db6db972232b392d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Apr 2014 22:38:59 -0700 Subject: [PATCH 005/352] Update MALLOCX_ARENA() documentation. Update MALLOCX_ARENA() documentation to no longer claim that it has no effect for huge region allocations. --- doc/jemalloc.xml.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index a4232409..4acb07f3 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -270,10 +270,10 @@ Use the arena specified by the index a (and by necessity bypass the thread - cache). This macro has no effect for huge regions, nor for regions - that were allocated via an arena other than the one specified. - This macro does not validate that a - specifies an arena index in the valid range. + cache). This macro has no effect for regions that were allocated + via an arena other than the one specified. This macro does not + validate that a specifies an arena index in + the valid range. From 644d414bc9ab52efbbf7ebeb350170106ec1f937 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 14 Apr 2014 22:49:23 -0700 Subject: [PATCH 006/352] Reverse the cc-silence default. Replace --enable-cc-silence with --disable-cc-silence, so that by default people won't see spurious warnings when building jemalloc. --- INSTALL | 8 ++++---- configure.ac | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/INSTALL b/INSTALL index 8530643d..55c919ae 100644 --- a/INSTALL +++ b/INSTALL @@ -71,10 +71,10 @@ any of the following arguments (not a definitive list) to 'configure': versions of jemalloc can coexist in the same installation directory. For example, libjemalloc.so.0 becomes libjemalloc.so.0. ---enable-cc-silence - Enable code that silences non-useful compiler warnings. This is helpful - when trying to tell serious warnings from those due to compiler - limitations, but it potentially incurs a performance penalty. +--disable-cc-silence + Disable code that silences non-useful compiler warnings. This is mainly + useful during development when auditing the set of warnings that are being + silenced. --enable-debug Enable assertions and validation code. This incurs a substantial diff --git a/configure.ac b/configure.ac index 04cefe95..b47d5723 100644 --- a/configure.ac +++ b/configure.ac @@ -577,18 +577,17 @@ cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.i cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in" cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:${srcroot}test/include/test/jemalloc_test_defs.h.in" -dnl Do not silence irrelevant compiler warnings by default, since enabling this -dnl option incurs a performance penalty. +dnl Silence irrelevant compiler warnings by default. AC_ARG_ENABLE([cc-silence], - [AS_HELP_STRING([--enable-cc-silence], - [Silence irrelevant compiler warnings])], + [AS_HELP_STRING([--disable-cc-silence], + [Do not silence irrelevant compiler warnings])], [if test "x$enable_cc_silence" = "xno" ; then enable_cc_silence="0" else enable_cc_silence="1" fi ], -[enable_cc_silence="0"] +[enable_cc_silence="1"] ) if test "x$enable_cc_silence" = "x1" ; then AC_DEFINE([JEMALLOC_CC_SILENCE], [ ]) From 4d434adb146375ad17f0d5e994ed5728d2942e3f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Apr 2014 12:09:48 -0700 Subject: [PATCH 007/352] Make dss non-optional, and fix an "arena..dss" mallctl bug. Make dss non-optional on all platforms which support sbrk(2). Fix the "arena..dss" mallctl to return an error if "primary" or "secondary" precedence is specified, but sbrk(2) is not supported. --- INSTALL | 4 --- configure.ac | 23 ++++----------- doc/jemalloc.xml.in | 29 +++++++++---------- include/jemalloc/internal/arena.h | 2 +- .../jemalloc/internal/jemalloc_internal.h.in | 2 +- .../internal/jemalloc_internal_defs.h.in | 3 -- src/arena.c | 5 +++- src/chunk.c | 8 ++--- src/chunk_dss.c | 20 ++++++------- src/ctl.c | 6 +--- src/huge.c | 6 ++-- test/integration/MALLOCX_ARENA.c | 17 +++++++++-- test/unit/mallctl.c | 20 ++++++++++--- 13 files changed, 72 insertions(+), 73 deletions(-) diff --git a/INSTALL b/INSTALL index 55c919ae..07f51d1e 100644 --- a/INSTALL +++ b/INSTALL @@ -145,10 +145,6 @@ any of the following arguments (not a definitive list) to 'configure': memory allocation algorithm that causes semi-permanent VM map holes under normal jemalloc operation. ---enable-dss - Enable support for page allocation/deallocation via sbrk(2), in addition to - mmap(2). - --disable-fill Disable support for junk/zero filling of memory, quarantine, and redzones. See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option diff --git a/configure.ac b/configure.ac index b47d5723..dc817e1c 100644 --- a/configure.ac +++ b/configure.ac @@ -836,34 +836,22 @@ if test "x$enable_munmap" = "x1" ; then fi AC_SUBST([enable_munmap]) -dnl Do not enable allocation from DSS by default. -AC_ARG_ENABLE([dss], - [AS_HELP_STRING([--enable-dss], [Enable allocation from DSS])], -[if test "x$enable_dss" = "xno" ; then - enable_dss="0" -else - enable_dss="1" -fi -], -[enable_dss="0"] -) +dnl Enable allocation from DSS if supported by the OS. +have_dss="1" dnl Check whether the BSD/SUSv1 sbrk() exists. If not, disable DSS support. AC_CHECK_FUNC([sbrk], [have_sbrk="1"], [have_sbrk="0"]) if test "x$have_sbrk" = "x1" ; then if test "x$sbrk_deprecated" == "x1" ; then AC_MSG_RESULT([Disabling dss allocation because sbrk is deprecated]) - enable_dss="0" - else - AC_DEFINE([JEMALLOC_HAVE_SBRK], [ ]) + have_dss="0" fi else - enable_dss="0" + have_dss="0" fi -if test "x$enable_dss" = "x1" ; then +if test "x$have_dss" = "x1" ; then AC_DEFINE([JEMALLOC_DSS], [ ]) fi -AC_SUBST([enable_dss]) dnl Support the junk/zero filling option by default. AC_ARG_ENABLE([fill], @@ -1461,7 +1449,6 @@ AC_MSG_RESULT([valgrind : ${enable_valgrind}]) AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) AC_MSG_RESULT([mremap : ${enable_mremap}]) AC_MSG_RESULT([munmap : ${enable_munmap}]) -AC_MSG_RESULT([dss : ${enable_dss}]) AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) AC_MSG_RESULT([tls : ${enable_tls}]) AC_MSG_RESULT([===============================================================================]) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 4acb07f3..16dd0bbe 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -448,8 +448,10 @@ for (i = 0; i < nbins; i++) { 2 to obtain memory, which is suboptimal for several reasons, including race conditions, increased fragmentation, and artificial limitations on maximum usable memory. If - is specified during configuration, this - allocator uses both mmap + sbrk + 2 is supported by the operating + system, this allocator uses both + mmap 2 and sbrk 2, in that order of preference; @@ -625,16 +627,6 @@ for (i = 0; i < nbins; i++) { build configuration. - - - config.dss - (bool) - r- - - was specified during - build configuration. - - config.fill @@ -790,10 +782,15 @@ for (i = 0; i < nbins; i++) { 2) allocation precedence as related to mmap 2 allocation. The following - settings are supported: “disabled”, “primary”, - and “secondary”. The default is “secondary” if - config.dss is - true, “disabled” otherwise. + settings are supported if + sbrk + 2 is supported by the operating + system: “disabled”, “primary”, and + “secondary”; otherwise only “disabled” is + supported. The default is “secondary” if + sbrk + 2 is supported by the operating + system; “disabled” otherwise. diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 0e14c2c4..6de312eb 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -434,7 +434,7 @@ void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc); dss_prec_t arena_dss_prec_get(arena_t *arena); -void arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); +bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index a374e2a3..4821b9bc 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -85,7 +85,7 @@ static const bool config_debug = false #endif ; -static const bool config_dss = +static const bool have_dss = #ifdef JEMALLOC_DSS true #else diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index c166fbd9..fc959671 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -76,9 +76,6 @@ */ #undef JEMALLOC_MUTEX_INIT_CB -/* Defined if sbrk() is supported. */ -#undef JEMALLOC_HAVE_SBRK - /* Non-empty if the tls_model attribute is supported. */ #undef JEMALLOC_TLS_MODEL diff --git a/src/arena.c b/src/arena.c index d5741000..8aa36fdf 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2243,13 +2243,16 @@ arena_dss_prec_get(arena_t *arena) return (ret); } -void +bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) { + if (have_dss == false) + return (dss_prec != dss_prec_disabled); malloc_mutex_lock(&arena->lock); arena->dss_prec = dss_prec; malloc_mutex_unlock(&arena->lock); + return (false); } void diff --git a/src/chunk.c b/src/chunk.c index 90ab116a..fdd693e0 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -153,7 +153,7 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, assert((alignment & chunksize_mask) == 0); /* "primary" dss. */ - if (config_dss && dss_prec == dss_prec_primary) { + if (have_dss && dss_prec == dss_prec_primary) { if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment, base, zero)) != NULL) goto label_return; @@ -167,7 +167,7 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) goto label_return; /* "secondary" dss. */ - if (config_dss && dss_prec == dss_prec_secondary) { + if (have_dss && dss_prec == dss_prec_secondary) { if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment, base, zero)) != NULL) goto label_return; @@ -305,7 +305,7 @@ chunk_unmap(void *chunk, size_t size) assert(size != 0); assert((size & chunksize_mask) == 0); - if (config_dss && chunk_in_dss(chunk)) + if (have_dss && chunk_in_dss(chunk)) chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size); else if (chunk_dealloc_mmap(chunk, size)) chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size); @@ -348,7 +348,7 @@ chunk_boot(void) return (true); memset(&stats_chunks, 0, sizeof(chunk_stats_t)); } - if (config_dss && chunk_dss_boot()) + if (have_dss && chunk_dss_boot()) return (true); extent_tree_szad_new(&chunks_szad_mmap); extent_tree_ad_new(&chunks_ad_mmap); diff --git a/src/chunk_dss.c b/src/chunk_dss.c index 510bb8be..36133f14 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -32,7 +32,7 @@ static void * chunk_dss_sbrk(intptr_t increment) { -#ifdef JEMALLOC_HAVE_SBRK +#ifdef JEMALLOC_DSS return (sbrk(increment)); #else not_implemented(); @@ -45,7 +45,7 @@ chunk_dss_prec_get(void) { dss_prec_t ret; - if (config_dss == false) + if (have_dss == false) return (dss_prec_disabled); malloc_mutex_lock(&dss_mtx); ret = dss_prec_default; @@ -57,8 +57,8 @@ bool chunk_dss_prec_set(dss_prec_t dss_prec) { - if (config_dss == false) - return (true); + if (have_dss == false) + return (dss_prec != dss_prec_disabled); malloc_mutex_lock(&dss_mtx); dss_prec_default = dss_prec; malloc_mutex_unlock(&dss_mtx); @@ -70,7 +70,7 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero) { void *ret; - cassert(config_dss); + cassert(have_dss); assert(size > 0 && (size & chunksize_mask) == 0); assert(alignment > 0 && (alignment & chunksize_mask) == 0); @@ -143,7 +143,7 @@ chunk_in_dss(void *chunk) { bool ret; - cassert(config_dss); + cassert(have_dss); malloc_mutex_lock(&dss_mtx); if ((uintptr_t)chunk >= (uintptr_t)dss_base @@ -160,7 +160,7 @@ bool chunk_dss_boot(void) { - cassert(config_dss); + cassert(have_dss); if (malloc_mutex_init(&dss_mtx)) return (true); @@ -175,7 +175,7 @@ void chunk_dss_prefork(void) { - if (config_dss) + if (have_dss) malloc_mutex_prefork(&dss_mtx); } @@ -183,7 +183,7 @@ void chunk_dss_postfork_parent(void) { - if (config_dss) + if (have_dss) malloc_mutex_postfork_parent(&dss_mtx); } @@ -191,7 +191,7 @@ void chunk_dss_postfork_child(void) { - if (config_dss) + if (have_dss) malloc_mutex_postfork_child(&dss_mtx); } diff --git a/src/ctl.c b/src/ctl.c index cc2c5aef..0340a274 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -74,7 +74,6 @@ CTL_PROTO(thread_allocatedp) CTL_PROTO(thread_deallocated) CTL_PROTO(thread_deallocatedp) CTL_PROTO(config_debug) -CTL_PROTO(config_dss) CTL_PROTO(config_fill) CTL_PROTO(config_lazy_lock) CTL_PROTO(config_mremap) @@ -213,7 +212,6 @@ static const ctl_named_node_t thread_node[] = { static const ctl_named_node_t config_node[] = { {NAME("debug"), CTL(config_debug)}, - {NAME("dss"), CTL(config_dss)}, {NAME("fill"), CTL(config_fill)}, {NAME("lazy_lock"), CTL(config_lazy_lock)}, {NAME("mremap"), CTL(config_mremap)}, @@ -1136,7 +1134,6 @@ label_return: /******************************************************************************/ CTL_RO_BOOL_CONFIG_GEN(config_debug) -CTL_RO_BOOL_CONFIG_GEN(config_dss) CTL_RO_BOOL_CONFIG_GEN(config_fill) CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) CTL_RO_BOOL_CONFIG_GEN(config_mremap) @@ -1356,8 +1353,7 @@ arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, arena_t *arena = arenas[arena_ind]; if (arena != NULL) { dss_prec_old = arena_dss_prec_get(arena); - arena_dss_prec_set(arena, dss_prec); - err = false; + err = arena_dss_prec_set(arena, dss_prec); } else err = true; } else { diff --git a/src/huge.c b/src/huge.c index d72f2135..e725fd90 100644 --- a/src/huge.c +++ b/src/huge.c @@ -140,7 +140,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * Use mremap(2) if this is a huge-->huge reallocation, and neither the * source nor the destination are in dss. */ - if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr) + if (oldsize >= chunksize && (have_dss == false || (chunk_in_dss(ptr) == false && chunk_in_dss(ret) == false))) { size_t newsize = huge_salloc(ret); @@ -198,12 +198,12 @@ static void huge_dalloc_junk(void *ptr, size_t usize) { - if (config_fill && config_dss && opt_junk) { + if (config_fill && have_dss && opt_junk) { /* * Only bother junk filling if the chunk isn't about to be * unmapped. */ - if (config_munmap == false || (config_dss && chunk_in_dss(ptr))) + if (config_munmap == false || (have_dss && chunk_in_dss(ptr))) memset(ptr, 0x5a, usize); } } diff --git a/test/integration/MALLOCX_ARENA.c b/test/integration/MALLOCX_ARENA.c index 695a5b66..30c203ae 100644 --- a/test/integration/MALLOCX_ARENA.c +++ b/test/integration/MALLOCX_ARENA.c @@ -2,6 +2,14 @@ #define NTHREADS 10 +static bool have_dss = +#ifdef JEMALLOC_DSS + true +#else + false +#endif + ; + void * thd_start(void *arg) { @@ -18,13 +26,16 @@ thd_start(void *arg) size_t mib[3]; size_t miblen = sizeof(mib) / sizeof(size_t); const char *dss_precs[] = {"disabled", "primary", "secondary"}; - const char *dss = dss_precs[thread_ind % - (sizeof(dss_precs)/sizeof(char*))]; + unsigned prec_ind = thread_ind % + (sizeof(dss_precs)/sizeof(char*)); + const char *dss = dss_precs[prec_ind]; + int expected_err = (have_dss || prec_ind == 0) ? 0 : EFAULT; assert_d_eq(mallctlnametomib("arena.0.dss", mib, &miblen), 0, "Error in mallctlnametomib()"); mib[1] = arena_ind; assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss, - sizeof(const char *)), 0, "Error in mallctlbymib()"); + sizeof(const char *)), expected_err, + "Error in mallctlbymib()"); } p = mallocx(1, MALLOCX_ARENA(arena_ind)); diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 31fb8105..caf20f86 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -127,7 +127,6 @@ TEST_BEGIN(test_mallctl_config) } while (0) TEST_MALLCTL_CONFIG(debug); - TEST_MALLCTL_CONFIG(dss); TEST_MALLCTL_CONFIG(fill); TEST_MALLCTL_CONFIG(lazy_lock); TEST_MALLCTL_CONFIG(mremap); @@ -255,15 +254,28 @@ TEST_BEGIN(test_arena_i_dss) { const char *dss_prec_old, *dss_prec_new; size_t sz = sizeof(dss_prec_old); + size_t mib[3]; + size_t miblen; - dss_prec_new = "primary"; - assert_d_eq(mallctl("arena.0.dss", &dss_prec_old, &sz, &dss_prec_new, + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("arena.0.dss", mib, &miblen), 0, + "Unexpected mallctlnametomib() error"); + + dss_prec_new = "disabled"; + assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, &dss_prec_new, sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure"); assert_str_ne(dss_prec_old, "primary", "Unexpected default for dss precedence"); - assert_d_eq(mallctl("arena.0.dss", &dss_prec_new, &sz, &dss_prec_old, + assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_new, &sz, &dss_prec_old, sizeof(dss_prec_old)), 0, "Unexpected mallctl() failure"); + + mib[1] = narenas_total_get(); + dss_prec_new = "disabled"; + assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, &dss_prec_new, + sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure"); + assert_str_ne(dss_prec_old, "primary", + "Unexpected default for dss precedence"); } TEST_END From a2c719b37445ce9083b6fc5084436dc37ceb7f75 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Apr 2014 12:46:28 -0700 Subject: [PATCH 008/352] Remove the "arenas.purge" mallctl. Remove the "arenas.purge" mallctl, which was obsoleted by the "arena..purge" mallctl in 3.1.0. --- doc/jemalloc.xml.in | 12 +----------- src/ctl.c | 27 --------------------------- test/unit/mallctl.c | 13 ------------- 3 files changed, 1 insertion(+), 51 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 16dd0bbe..b0c77c20 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1267,7 +1267,7 @@ malloc_conf = "xmalloc:true";]]> arena.<i>.purge - (unsigned) + (void) -- Purge unused dirty pages for arena <i>, or for @@ -1410,16 +1410,6 @@ malloc_conf = "xmalloc:true";]]> class. - - - arenas.purge - (unsigned) - -w - - Purge unused dirty pages for the specified arena, or - for all arenas if none is specified. - - arenas.extend diff --git a/src/ctl.c b/src/ctl.c index 0340a274..3d44a951 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -129,7 +129,6 @@ CTL_PROTO(arenas_tcache_max) CTL_PROTO(arenas_nbins) CTL_PROTO(arenas_nhbins) CTL_PROTO(arenas_nlruns) -CTL_PROTO(arenas_purge) CTL_PROTO(arenas_extend) CTL_PROTO(prof_active) CTL_PROTO(prof_dump) @@ -301,7 +300,6 @@ static const ctl_named_node_t arenas_node[] = { {NAME("bin"), CHILD(indexed, arenas_bin)}, {NAME("nlruns"), CTL(arenas_nlruns)}, {NAME("lrun"), CHILD(indexed, arenas_lrun)}, - {NAME("purge"), CTL(arenas_purge)}, {NAME("extend"), CTL(arenas_extend)} }; @@ -1468,31 +1466,6 @@ arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) return (super_arenas_lrun_i_node); } -static int -arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - unsigned arena_ind; - - malloc_mutex_lock(&ctl_mtx); - WRITEONLY(); - arena_ind = UINT_MAX; - WRITE(arena_ind, unsigned); - if (newp != NULL && arena_ind >= ctl_stats.narenas) - ret = EFAULT; - else { - if (arena_ind == UINT_MAX) - arena_ind = ctl_stats.narenas; - arena_purge(arena_ind); - ret = 0; - } - -label_return: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - static int arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index caf20f86..73f42ddd 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -279,18 +279,6 @@ TEST_BEGIN(test_arena_i_dss) } TEST_END -TEST_BEGIN(test_arenas_purge) -{ - unsigned arena = 0; - - assert_d_eq(mallctl("arenas.purge", NULL, NULL, &arena, sizeof(arena)), - 0, "Unexpected mallctl() failure"); - - assert_d_eq(mallctl("arenas.purge", NULL, NULL, NULL, 0), 0, - "Unexpected mallctl() failure"); -} -TEST_END - TEST_BEGIN(test_arenas_initialized) { unsigned narenas; @@ -417,7 +405,6 @@ main(void) test_thread_arena, test_arena_i_purge, test_arena_i_dss, - test_arenas_purge, test_arenas_initialized, test_arenas_constants, test_arenas_bin_constants, From ecd3e59ca351d7111ec72a327fe0c009f2aa69a0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Apr 2014 14:33:50 -0700 Subject: [PATCH 009/352] Remove the "opt.valgrind" mallctl. Remove the "opt.valgrind" mallctl because it is unnecessary -- jemalloc automatically detects whether it is running inside valgrind. --- doc/jemalloc.xml.in | 13 ------- .../jemalloc/internal/jemalloc_internal.h.in | 9 ++--- include/jemalloc/internal/private_symbols.txt | 2 +- src/ctl.c | 3 -- src/jemalloc.c | 35 +++++++++---------- src/quarantine.c | 2 +- test/unit/mallctl.c | 1 - 7 files changed, 24 insertions(+), 41 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index b0c77c20..3e0b806f 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -943,19 +943,6 @@ for (i = 0; i < nbins; i++) { is disabled by default. - - - opt.valgrind - (bool) - r- - [] - - Valgrind - support enabled/disabled. This option is vestigal because jemalloc - auto-detects whether it is running inside Valgrind. This option is - disabled by default, unless running inside Valgrind. - - opt.xmalloc diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 4821b9bc..9b1a6c85 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -377,12 +377,12 @@ static const bool config_ivsalloc = * usable space. */ #define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ - if (config_valgrind && opt_valgrind && cond) \ + if (config_valgrind && in_valgrind && cond) \ VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ } while (0) #define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ old_rzsize, zero) do { \ - if (config_valgrind && opt_valgrind) { \ + if (config_valgrind && in_valgrind) { \ size_t rzsize = p2rz(ptr); \ \ if (ptr == old_ptr) { \ @@ -418,7 +418,7 @@ static const bool config_ivsalloc = } \ } while (0) #define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ - if (config_valgrind && opt_valgrind) \ + if (config_valgrind && in_valgrind) \ VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \ } while (0) #else @@ -504,11 +504,12 @@ extern bool opt_junk; extern size_t opt_quarantine; extern bool opt_redzone; extern bool opt_utrace; -extern bool opt_valgrind; extern bool opt_xmalloc; extern bool opt_zero; extern size_t opt_narenas; +extern bool in_valgrind; + /* Number of CPUs. */ extern unsigned ncpus; diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index e1cb28f8..c1403779 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -217,6 +217,7 @@ idalloc idalloct imalloc imalloct +in_valgrind ipalloc ipalloct iqalloc @@ -278,7 +279,6 @@ opt_redzone opt_stats_print opt_tcache opt_utrace -opt_valgrind opt_xmalloc opt_zero p2rz diff --git a/src/ctl.c b/src/ctl.c index 3d44a951..9ee5de9f 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -98,7 +98,6 @@ CTL_PROTO(opt_zero) CTL_PROTO(opt_quarantine) CTL_PROTO(opt_redzone) CTL_PROTO(opt_utrace) -CTL_PROTO(opt_valgrind) CTL_PROTO(opt_xmalloc) CTL_PROTO(opt_tcache) CTL_PROTO(opt_lg_tcache_max) @@ -238,7 +237,6 @@ static const ctl_named_node_t opt_node[] = { {NAME("quarantine"), CTL(opt_quarantine)}, {NAME("redzone"), CTL(opt_redzone)}, {NAME("utrace"), CTL(opt_utrace)}, - {NAME("valgrind"), CTL(opt_valgrind)}, {NAME("xmalloc"), CTL(opt_xmalloc)}, {NAME("tcache"), CTL(opt_tcache)}, {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, @@ -1159,7 +1157,6 @@ CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) -CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool) CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) diff --git a/src/jemalloc.c b/src/jemalloc.c index 0de59408..11f1c450 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -27,11 +27,13 @@ bool opt_junk = size_t opt_quarantine = ZU(0); bool opt_redzone = false; bool opt_utrace = false; -bool opt_valgrind = false; bool opt_xmalloc = false; bool opt_zero = false; size_t opt_narenas = 0; +/* Initialized to true if the process is running inside Valgrind. */ +bool in_valgrind; + unsigned ncpus; malloc_mutex_t arenas_lock; @@ -394,14 +396,14 @@ malloc_conf_init(void) * valgrind option remains in jemalloc 3.x for compatibility reasons. */ if (config_valgrind) { - opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; - if (config_fill && opt_valgrind) { + in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; + if (config_fill && in_valgrind) { opt_junk = false; assert(opt_zero == false); opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; opt_redzone = true; } - if (config_tcache && opt_valgrind) + if (config_tcache && in_valgrind) opt_tcache = false; } @@ -608,9 +610,6 @@ malloc_conf_init(void) if (config_utrace) { CONF_HANDLE_BOOL(opt_utrace, "utrace") } - if (config_valgrind) { - CONF_HANDLE_BOOL(opt_valgrind, "valgrind") - } if (config_xmalloc) { CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") } @@ -910,7 +909,7 @@ imalloc_prof(size_t usize, prof_thr_cnt_t *cnt) ret = imalloc_prof(usize, cnt); \ } else { \ if (config_stats || (config_valgrind && \ - opt_valgrind)) \ + in_valgrind)) \ usize = s2u(size); \ ret = imalloc(size); \ } \ @@ -1153,7 +1152,7 @@ je_calloc(size_t num, size_t size) PROF_ALLOC_PREP(1, usize, cnt); ret = icalloc_prof(usize, cnt); } else { - if (config_stats || (config_valgrind && opt_valgrind)) + if (config_stats || (config_valgrind && in_valgrind)) usize = s2u(num_size); ret = icalloc(num_size); } @@ -1228,7 +1227,7 @@ ifree(void *ptr) usize = isalloc(ptr, config_prof); if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - if (config_valgrind && opt_valgrind) + if (config_valgrind && in_valgrind) rzsize = p2rz(ptr); iqalloc(ptr); JEMALLOC_VALGRIND_FREE(ptr, rzsize); @@ -1257,9 +1256,9 @@ je_realloc(void *ptr, size_t size) malloc_thread_init(); if ((config_prof && opt_prof) || config_stats || - (config_valgrind && opt_valgrind)) + (config_valgrind && in_valgrind)) old_usize = isalloc(ptr, config_prof); - if (config_valgrind && opt_valgrind) + if (config_valgrind && in_valgrind) old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); if (config_prof && opt_prof) { @@ -1269,7 +1268,7 @@ je_realloc(void *ptr, size_t size) PROF_ALLOC_PREP(1, usize, cnt); ret = irealloc_prof(ptr, old_usize, usize, cnt); } else { - if (config_stats || (config_valgrind && opt_valgrind)) + if (config_stats || (config_valgrind && in_valgrind)) usize = s2u(size); ret = iralloc(ptr, size, 0, 0, false); } @@ -1574,9 +1573,9 @@ je_rallocx(void *ptr, size_t size, int flags) } if ((config_prof && opt_prof) || config_stats || - (config_valgrind && opt_valgrind)) + (config_valgrind && in_valgrind)) old_usize = isalloc(ptr, config_prof); - if (config_valgrind && opt_valgrind) + if (config_valgrind && in_valgrind) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { @@ -1594,7 +1593,7 @@ je_rallocx(void *ptr, size_t size, int flags) try_tcache_dalloc, arena); if (p == NULL) goto label_oom; - if (config_stats || (config_valgrind && opt_valgrind)) + if (config_stats || (config_valgrind && in_valgrind)) usize = isalloc(p, config_prof); } @@ -1702,7 +1701,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) arena = NULL; old_usize = isalloc(ptr, config_prof); - if (config_valgrind && opt_valgrind) + if (config_valgrind && in_valgrind) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { @@ -1784,7 +1783,7 @@ je_dallocx(void *ptr, int flags) } if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - if (config_valgrind && opt_valgrind) + if (config_valgrind && in_valgrind) rzsize = p2rz(ptr); iqalloct(ptr, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); diff --git a/src/quarantine.c b/src/quarantine.c index 54315116..3b874422 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -146,7 +146,7 @@ quarantine(void *ptr) * Only do redzone validation if Valgrind isn't in * operation. */ - if ((config_valgrind == false || opt_valgrind == false) + if ((config_valgrind == false || in_valgrind == false) && usize <= SMALL_MAXCLASS) arena_quarantine_junk_small(ptr, usize); else diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 73f42ddd..754834c1 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -170,7 +170,6 @@ TEST_BEGIN(test_mallctl_opt) TEST_MALLCTL_OPT(bool, redzone, fill); TEST_MALLCTL_OPT(bool, zero, fill); TEST_MALLCTL_OPT(bool, utrace, utrace); - TEST_MALLCTL_OPT(bool, valgrind, valgrind); TEST_MALLCTL_OPT(bool, xmalloc, xmalloc); TEST_MALLCTL_OPT(bool, tcache, tcache); TEST_MALLCTL_OPT(size_t, lg_tcache_max, tcache); From bd87b01999416ec7418ff8bdb504d9b6c009ff68 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 15 Apr 2014 16:35:08 -0700 Subject: [PATCH 010/352] Optimize Valgrind integration. Forcefully disable tcache if running inside Valgrind, and remove Valgrind calls in tcache-specific code. Restructure Valgrind-related code to move most Valgrind calls out of the fast path functions. Take advantage of static knowledge to elide some branches in JEMALLOC_VALGRIND_REALLOC(). --- Makefile.in | 4 + doc/jemalloc.xml.in | 3 +- .../jemalloc/internal/jemalloc_internal.h.in | 84 +------------ include/jemalloc/internal/private_symbols.txt | 4 + include/jemalloc/internal/tcache.h | 7 +- include/jemalloc/internal/valgrind.h | 112 ++++++++++++++++++ src/arena.c | 28 ++--- src/base.c | 7 +- src/chunk.c | 6 +- src/chunk_dss.c | 3 +- src/jemalloc.c | 75 +++++++----- src/valgrind.c | 34 ++++++ 12 files changed, 231 insertions(+), 136 deletions(-) create mode 100644 include/jemalloc/internal/valgrind.h create mode 100644 src/valgrind.c diff --git a/Makefile.in b/Makefile.in index f7aa7d87..e411804a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -48,6 +48,7 @@ cfgoutputs_in := @cfgoutputs_in@ cfgoutputs_out := @cfgoutputs_out@ enable_autogen := @enable_autogen@ enable_code_coverage := @enable_code_coverage@ +enable_valgrind := @enable_valgrind@ enable_zone_allocator := @enable_zone_allocator@ DSO_LDFLAGS = @DSO_LDFLAGS@ SOREV = @SOREV@ @@ -82,6 +83,9 @@ C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \ $(srcroot)src/mb.c $(srcroot)src/mutex.c $(srcroot)src/prof.c \ $(srcroot)src/quarantine.c $(srcroot)src/rtree.c $(srcroot)src/stats.c \ $(srcroot)src/tcache.c $(srcroot)src/util.c $(srcroot)src/tsd.c +ifeq ($(enable_valgrind), 1) +C_SRCS += $(srcroot)src/valgrind.c +endif ifeq ($(enable_zone_allocator), 1) C_SRCS += $(srcroot)src/zone.c endif diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 3e0b806f..78e9b3c6 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -979,7 +979,8 @@ malloc_conf = "xmalloc:true";]]> linkend="opt.lg_tcache_max">opt.lg_tcache_max option for related tuning information. This option is enabled by default unless running inside Valgrind. + url="http://valgrind.org/">Valgrind, in which case it is + forcefully disabled. diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 9b1a6c85..50d44cc2 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -60,11 +60,6 @@ typedef intptr_t ssize_t; #include #endif -#ifdef JEMALLOC_VALGRIND -#include -#include -#endif - #define JEMALLOC_NO_DEMANGLE #ifdef JEMALLOC_JET # define JEMALLOC_N(n) jet_##n @@ -362,81 +357,7 @@ static const bool config_ivsalloc = # define VARIABLE_ARRAY(type, name, count) type name[count] #endif -#ifdef JEMALLOC_VALGRIND -/* - * The JEMALLOC_VALGRIND_*() macros must be macros rather than functions - * so that when Valgrind reports errors, there are no extra stack frames - * in the backtraces. - * - * The size that is reported to valgrind must be consistent through a chain of - * malloc..realloc..realloc calls. Request size isn't recorded anywhere in - * jemalloc, so it is critical that all callers of these macros provide usize - * rather than request size. As a result, buffer overflow detection is - * technically weakened for the standard API, though it is generally accepted - * practice to consider any extra bytes reported by malloc_usable_size() as - * usable space. - */ -#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ - if (config_valgrind && in_valgrind && cond) \ - VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ -} while (0) -#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ - old_rzsize, zero) do { \ - if (config_valgrind && in_valgrind) { \ - size_t rzsize = p2rz(ptr); \ - \ - if (ptr == old_ptr) { \ - VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ - usize, rzsize); \ - if (zero && old_usize < usize) { \ - VALGRIND_MAKE_MEM_DEFINED( \ - (void *)((uintptr_t)ptr + \ - old_usize), usize - old_usize); \ - } \ - } else { \ - if (old_ptr != NULL) { \ - VALGRIND_FREELIKE_BLOCK(old_ptr, \ - old_rzsize); \ - } \ - if (ptr != NULL) { \ - size_t copy_size = (old_usize < usize) \ - ? old_usize : usize; \ - size_t tail_size = usize - copy_size; \ - VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \ - rzsize, false); \ - if (copy_size > 0) { \ - VALGRIND_MAKE_MEM_DEFINED(ptr, \ - copy_size); \ - } \ - if (zero && tail_size > 0) { \ - VALGRIND_MAKE_MEM_DEFINED( \ - (void *)((uintptr_t)ptr + \ - copy_size), tail_size); \ - } \ - } \ - } \ - } \ -} while (0) -#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ - if (config_valgrind && in_valgrind) \ - VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \ -} while (0) -#else -#define RUNNING_ON_VALGRIND ((unsigned)0) -#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \ - do {} while (0) -#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \ - do {} while (0) -#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0) -#define VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0) -#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0) -#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0) -#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0) -#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ - old_rzsize, zero) do {} while (0) -#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0) -#endif - +#include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" @@ -463,6 +384,7 @@ static const bool config_ivsalloc = /******************************************************************************/ #define JEMALLOC_H_STRUCTS +#include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" @@ -534,6 +456,7 @@ void jemalloc_prefork(void); void jemalloc_postfork_parent(void); void jemalloc_postfork_child(void); +#include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" @@ -560,6 +483,7 @@ void jemalloc_postfork_child(void); /******************************************************************************/ #define JEMALLOC_H_INLINES +#include "jemalloc/internal/valgrind.h" #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/prng.h" diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index c1403779..9d77cbaa 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -409,3 +409,7 @@ thread_allocated_tsd_set tsd_init_check_recursion tsd_init_finish u2rz +valgrind_freelike_block +valgrind_make_mem_defined +valgrind_make_mem_noaccess +valgrind_make_mem_undefined diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 51974136..af248451 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -314,13 +314,11 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) } else if (opt_zero) memset(ret, 0, size); } - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { if (config_fill && opt_junk) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); } @@ -369,11 +367,8 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) else if (opt_zero) memset(ret, 0, size); } - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); - } else { - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + } else memset(ret, 0, size); - } if (config_stats) tbin->tstats.nrequests++; diff --git a/include/jemalloc/internal/valgrind.h b/include/jemalloc/internal/valgrind.h new file mode 100644 index 00000000..52c93f29 --- /dev/null +++ b/include/jemalloc/internal/valgrind.h @@ -0,0 +1,112 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#ifdef JEMALLOC_VALGRIND +#include + +/* + * The size that is reported to Valgrind must be consistent through a chain of + * malloc..realloc..realloc calls. Request size isn't recorded anywhere in + * jemalloc, so it is critical that all callers of these macros provide usize + * rather than request size. As a result, buffer overflow detection is + * technically weakened for the standard API, though it is generally accepted + * practice to consider any extra bytes reported by malloc_usable_size() as + * usable space. + */ +#define JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(ptr, usize) do { \ + if (in_valgrind) \ + valgrind_make_mem_noaccess(ptr, usize); \ +} while (0) +#define JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize) do { \ + if (in_valgrind) \ + valgrind_make_mem_undefined(ptr, usize); \ +} while (0) +#define JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ptr, usize) do { \ + if (in_valgrind) \ + valgrind_make_mem_defined(ptr, usize); \ +} while (0) +/* + * The VALGRIND_MALLOCLIKE_BLOCK() and VALGRIND_RESIZEINPLACE_BLOCK() macro + * calls must be embedded in macros rather than in functions so that when + * Valgrind reports errors, there are no extra stack frames in the backtraces. + */ +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ + if (in_valgrind && cond) \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ +} while (0) +#define JEMALLOC_VALGRIND_REALLOC(maybe_moved, ptr, usize, \ + ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \ + zero) do { \ + if (in_valgrind) { \ + size_t rzsize = p2rz(ptr); \ + \ + if (!maybe_moved || ptr == old_ptr) { \ + VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ + usize, rzsize); \ + if (zero && old_usize < usize) { \ + valgrind_make_mem_defined( \ + (void *)((uintptr_t)ptr + \ + old_usize), usize - old_usize); \ + } \ + } else { \ + if (!old_ptr_maybe_null || old_ptr != NULL) { \ + valgrind_freelike_block(old_ptr, \ + old_rzsize); \ + } \ + if (!ptr_maybe_null || ptr != NULL) { \ + size_t copy_size = (old_usize < usize) \ + ? old_usize : usize; \ + size_t tail_size = usize - copy_size; \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \ + rzsize, false); \ + if (copy_size > 0) { \ + valgrind_make_mem_defined(ptr, \ + copy_size); \ + } \ + if (zero && tail_size > 0) { \ + valgrind_make_mem_defined( \ + (void *)((uintptr_t)ptr + \ + copy_size), tail_size); \ + } \ + } \ + } \ + } \ +} while (0) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ + if (in_valgrind) \ + valgrind_freelike_block(ptr, rzsize); \ +} while (0) +#else +#define RUNNING_ON_VALGRIND ((unsigned)0) +#define JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(ptr, usize) do {} while (0) +#define JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize) do {} while (0) +#define JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ptr, usize) do {} while (0) +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0) +#define JEMALLOC_VALGRIND_REALLOC(maybe_moved, ptr, usize, \ + ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \ + zero) do {} while (0) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0) +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#ifdef JEMALLOC_VALGRIND +void valgrind_make_mem_noaccess(void *ptr, size_t usize); +void valgrind_make_mem_undefined(void *ptr, size_t usize); +void valgrind_make_mem_defined(void *ptr, size_t usize); +void valgrind_freelike_block(void *ptr, size_t usize); +#endif + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/src/arena.c b/src/arena.c index 8aa36fdf..3952e70d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -337,8 +337,8 @@ static inline void arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) { - VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << - LG_PAGE)), (npages << LG_PAGE)); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (npages << LG_PAGE)); memset((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), 0, (npages << LG_PAGE)); } @@ -347,8 +347,8 @@ static inline void arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) { - VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind << - LG_PAGE)), PAGE); + JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)((uintptr_t)chunk + (run_ind + << LG_PAGE)), PAGE); } static inline void @@ -457,7 +457,7 @@ arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, arena_run_zero(chunk, run_ind, need_pages); } } else { - VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } @@ -525,7 +525,7 @@ arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) == 0) arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); - VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } @@ -592,14 +592,14 @@ arena_chunk_init_hard(arena_t *arena) * the chunk is not zeroed. */ if (zero == false) { - VALGRIND_MAKE_MEM_UNDEFINED((void *)arena_mapp_get(chunk, - map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, - chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, - map_bias+1))); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( + (void *)arena_mapp_get(chunk, map_bias+1), + (size_t)((uintptr_t) arena_mapp_get(chunk, chunk_npages-1) - + (uintptr_t)arena_mapp_get(chunk, map_bias+1))); for (i = map_bias+1; i < chunk_npages-1; i++) arena_mapbits_unzeroed_set(chunk, i, unzeroed); } else { - VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, + JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, map_bias+1))); @@ -1645,13 +1645,13 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) } else if (opt_zero) memset(ret, 0, size); } - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { if (config_fill && opt_junk) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); } @@ -2226,7 +2226,7 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, * expectation that the extra bytes will be reliably preserved. */ copysize = (size < oldsize) ? size : oldsize; - VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); iqalloct(ptr, try_tcache_dalloc); return (ret); diff --git a/src/base.c b/src/base.c index 4e62e8fa..03dcf8f4 100644 --- a/src/base.c +++ b/src/base.c @@ -63,7 +63,7 @@ base_alloc(size_t size) ret = base_next_addr; base_next_addr = (void *)((uintptr_t)base_next_addr + csize); malloc_mutex_unlock(&base_mtx); - VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); return (ret); } @@ -89,7 +89,8 @@ base_node_alloc(void) ret = base_nodes; base_nodes = *(extent_node_t **)ret; malloc_mutex_unlock(&base_mtx); - VALGRIND_MAKE_MEM_UNDEFINED(ret, sizeof(extent_node_t)); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, + sizeof(extent_node_t)); } else { malloc_mutex_unlock(&base_mtx); ret = (extent_node_t *)base_alloc(sizeof(extent_node_t)); @@ -102,7 +103,7 @@ void base_node_dealloc(extent_node_t *node) { - VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); malloc_mutex_lock(&base_mtx); *(extent_node_t **)node = base_nodes; base_nodes = node; diff --git a/src/chunk.c b/src/chunk.c index fdd693e0..246324a2 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -127,7 +127,7 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, size_t i; size_t *p = (size_t *)(uintptr_t)ret; - VALGRIND_MAKE_MEM_DEFINED(ret, size); + JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ret, size); for (i = 0; i < size / sizeof(size_t); i++) assert(p[i] == 0); } @@ -203,7 +203,7 @@ label_return: prof_gdump(); } if (config_valgrind) - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } assert(CHUNK_ADDR2BASE(ret) == ret); return (ret); @@ -217,7 +217,7 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, extent_node_t *xnode, *node, *prev, *xprev, key; unzeroed = pages_purge(chunk, size); - VALGRIND_MAKE_MEM_NOACCESS(chunk, size); + JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); /* * Allocate a node before acquiring chunks_mtx even though it might not diff --git a/src/chunk_dss.c b/src/chunk_dss.c index 36133f14..82faf918 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -126,7 +126,8 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero) if (cpad_size != 0) chunk_unmap(cpad, cpad_size); if (*zero) { - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( + ret, size); memset(ret, 0, size); } return (ret); diff --git a/src/jemalloc.c b/src/jemalloc.c index 11f1c450..36eae722 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -479,9 +479,10 @@ malloc_conf_init(void) while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, &vlen) == false) { -#define CONF_HANDLE_BOOL(o, n) \ - if (sizeof(n)-1 == klen && strncmp(n, k, \ - klen) == 0) { \ +#define CONF_MATCH(n) \ + (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) +#define CONF_HANDLE_BOOL(o, n, cont) \ + if (CONF_MATCH(n)) { \ if (strncmp("true", v, vlen) == 0 && \ vlen == sizeof("true")-1) \ o = true; \ @@ -493,11 +494,11 @@ malloc_conf_init(void) "Invalid conf value", \ k, klen, v, vlen); \ } \ - continue; \ + if (cont) \ + continue; \ } #define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ - if (sizeof(n)-1 == klen && strncmp(n, k, \ - klen) == 0) { \ + if (CONF_MATCH(n)) { \ uintmax_t um; \ char *end; \ \ @@ -528,8 +529,7 @@ malloc_conf_init(void) continue; \ } #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ - if (sizeof(n)-1 == klen && strncmp(n, k, \ - klen) == 0) { \ + if (CONF_MATCH(n)) { \ long l; \ char *end; \ \ @@ -550,8 +550,7 @@ malloc_conf_init(void) continue; \ } #define CONF_HANDLE_CHAR_P(o, n, d) \ - if (sizeof(n)-1 == klen && strncmp(n, k, \ - klen) == 0) { \ + if (CONF_MATCH(n)) { \ size_t cpylen = (vlen <= \ sizeof(o)-1) ? vlen : \ sizeof(o)-1; \ @@ -560,7 +559,7 @@ malloc_conf_init(void) continue; \ } - CONF_HANDLE_BOOL(opt_abort, "abort") + CONF_HANDLE_BOOL(opt_abort, "abort", true) /* * Chunks always require at least one header page, plus * one data page in the absence of redzones, or three @@ -599,44 +598,62 @@ malloc_conf_init(void) SIZE_T_MAX, false) CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", -1, (sizeof(size_t) << 3) - 1) - CONF_HANDLE_BOOL(opt_stats_print, "stats_print") + CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) if (config_fill) { - CONF_HANDLE_BOOL(opt_junk, "junk") + CONF_HANDLE_BOOL(opt_junk, "junk", true) CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 0, SIZE_T_MAX, false) - CONF_HANDLE_BOOL(opt_redzone, "redzone") - CONF_HANDLE_BOOL(opt_zero, "zero") + CONF_HANDLE_BOOL(opt_redzone, "redzone", true) + CONF_HANDLE_BOOL(opt_zero, "zero", true) } if (config_utrace) { - CONF_HANDLE_BOOL(opt_utrace, "utrace") + CONF_HANDLE_BOOL(opt_utrace, "utrace", true) } if (config_xmalloc) { - CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") + CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc", true) } if (config_tcache) { - CONF_HANDLE_BOOL(opt_tcache, "tcache") + CONF_HANDLE_BOOL(opt_tcache, "tcache", + !config_valgrind || !in_valgrind) + if (CONF_MATCH("tcache")) { + assert(config_valgrind && in_valgrind); + if (opt_tcache) { + opt_tcache = false; + malloc_conf_error( + "tcache cannot be enabled " + "while running inside Valgrind", + k, klen, v, vlen); + } + continue; + } CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", -1, (sizeof(size_t) << 3) - 1) } if (config_prof) { - CONF_HANDLE_BOOL(opt_prof, "prof") + CONF_HANDLE_BOOL(opt_prof, "prof", true) CONF_HANDLE_CHAR_P(opt_prof_prefix, "prof_prefix", "jeprof") - CONF_HANDLE_BOOL(opt_prof_active, "prof_active") + CONF_HANDLE_BOOL(opt_prof_active, "prof_active", + true) CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, "lg_prof_sample", 0, (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") + CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", + true) CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, "lg_prof_interval", -1, (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") - CONF_HANDLE_BOOL(opt_prof_final, "prof_final") - CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") + CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump", + true) + CONF_HANDLE_BOOL(opt_prof_final, "prof_final", + true) + CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak", + true) } malloc_conf_error("Invalid conf pair", k, klen, v, vlen); +#undef CONF_MATCH #undef CONF_HANDLE_BOOL #undef CONF_HANDLE_SIZE_T #undef CONF_HANDLE_SSIZE_T @@ -1293,8 +1310,8 @@ je_realloc(void *ptr, size_t size) ta->deallocated += old_usize; } UTRACE(ptr, size, ret); - JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_usize, old_rzsize, - false); + JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, + old_rzsize, true, false); return (ret); } @@ -1604,7 +1621,8 @@ je_rallocx(void *ptr, size_t size, int flags) ta->deallocated += old_usize; } UTRACE(ptr, size, p); - JEMALLOC_VALGRIND_REALLOC(p, usize, ptr, old_usize, old_rzsize, zero); + JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize, + old_rzsize, false, zero); return (p); label_oom: if (config_xmalloc && opt_xmalloc) { @@ -1731,7 +1749,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) ta->allocated += usize; ta->deallocated += old_usize; } - JEMALLOC_VALGRIND_REALLOC(ptr, usize, ptr, old_usize, old_rzsize, zero); + JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize, + old_rzsize, false, zero); label_not_resized: UTRACE(ptr, size, ptr); return (usize); diff --git a/src/valgrind.c b/src/valgrind.c new file mode 100644 index 00000000..8e7ef3a2 --- /dev/null +++ b/src/valgrind.c @@ -0,0 +1,34 @@ +#include "jemalloc/internal/jemalloc_internal.h" +#ifndef JEMALLOC_VALGRIND +# error "This source file is for Valgrind integration." +#endif + +#include + +void +valgrind_make_mem_noaccess(void *ptr, size_t usize) +{ + + VALGRIND_MAKE_MEM_NOACCESS(ptr, usize); +} + +void +valgrind_make_mem_undefined(void *ptr, size_t usize) +{ + + VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize); +} + +void +valgrind_make_mem_defined(void *ptr, size_t usize) +{ + + VALGRIND_MAKE_MEM_DEFINED(ptr, usize); +} + +void +valgrind_freelike_block(void *ptr, size_t usize) +{ + + VALGRIND_FREELIKE_BLOCK(ptr, usize); +} From a7619b7fa56f98d1ca99a23b458696dd37c12b77 Mon Sep 17 00:00:00 2001 From: Ben Maurer Date: Tue, 15 Apr 2014 13:28:37 -0700 Subject: [PATCH 011/352] outline rare tcache_get codepaths --- include/jemalloc/internal/private_symbols.txt | 1 + include/jemalloc/internal/tcache.h | 35 +--------------- src/tcache.c | 40 +++++++++++++++++++ 3 files changed, 43 insertions(+), 33 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index f52d49f9..376f95d1 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -384,6 +384,7 @@ tcache_event tcache_event_hard tcache_flush tcache_get +tcache_get_hard tcache_initialized tcache_maxclass tcache_salloc diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 51974136..96447f42 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -110,6 +110,7 @@ void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_associate(tcache_t *tcache, arena_t *arena); void tcache_arena_dissociate(tcache_t *tcache); +tcache_t *tcache_get_hard(tcache_t *tcache, bool create); tcache_t *tcache_create(arena_t *arena); void tcache_destroy(tcache_t *tcache); void tcache_thread_cleanup(void *arg); @@ -220,39 +221,7 @@ tcache_get(bool create) if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) { if (tcache == TCACHE_STATE_DISABLED) return (NULL); - if (tcache == NULL) { - if (create == false) { - /* - * Creating a tcache here would cause - * allocation as a side effect of free(). - * Ordinarily that would be okay since - * tcache_create() failure is a soft failure - * that doesn't propagate. However, if TLS - * data are freed via free() as in glibc, - * subtle corruption could result from setting - * a TLS variable after its backing memory is - * freed. - */ - return (NULL); - } - if (tcache_enabled_get() == false) { - tcache_enabled_set(false); /* Memoize. */ - return (NULL); - } - return (tcache_create(choose_arena(NULL))); - } - if (tcache == TCACHE_STATE_PURGATORY) { - /* - * Make a note that an allocator function was called - * after tcache_thread_cleanup() was called. - */ - tcache = TCACHE_STATE_REINCARNATED; - tcache_tsd_set(&tcache); - return (NULL); - } - if (tcache == TCACHE_STATE_REINCARNATED) - return (NULL); - not_reached(); + tcache = tcache_get_hard(tcache, create); } return (tcache); diff --git a/src/tcache.c b/src/tcache.c index 6de92960..868f2d77 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -265,6 +265,46 @@ tcache_arena_dissociate(tcache_t *tcache) } } +tcache_t * +tcache_get_hard(tcache_t *tcache, bool create) +{ + + if (tcache == NULL) { + if (create == false) { + /* + * Creating a tcache here would cause + * allocation as a side effect of free(). + * Ordinarily that would be okay since + * tcache_create() failure is a soft failure + * that doesn't propagate. However, if TLS + * data are freed via free() as in glibc, + * subtle corruption could result from setting + * a TLS variable after its backing memory is + * freed. + */ + return (NULL); + } + if (tcache_enabled_get() == false) { + tcache_enabled_set(false); /* Memoize. */ + return (NULL); + } + return (tcache_create(choose_arena(NULL))); + } + if (tcache == TCACHE_STATE_PURGATORY) { + /* + * Make a note that an allocator function was called + * after tcache_thread_cleanup() was called. + */ + tcache = TCACHE_STATE_REINCARNATED; + tcache_tsd_set(&tcache); + return (NULL); + } + if (tcache == TCACHE_STATE_REINCARNATED) + return (NULL); + not_reached(); + return (NULL); +} + tcache_t * tcache_create(arena_t *arena) { From 6c39f9e059d0825f4c29d8cec9f318b798912c3c Mon Sep 17 00:00:00 2001 From: Ben Maurer Date: Tue, 15 Apr 2014 13:47:13 -0700 Subject: [PATCH 012/352] refactor profiling. only use a bytes till next sample variable. --- include/jemalloc/internal/private_symbols.txt | 1 + include/jemalloc/internal/prof.h | 222 ++++++------------ src/prof.c | 65 ++++- 3 files changed, 134 insertions(+), 154 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 376f95d1..032bed4f 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -299,6 +299,7 @@ prof_idump prof_interval prof_lookup prof_malloc +prof_malloc_record_object prof_mdump prof_postfork_child prof_postfork_parent diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 56014f18..27be10c1 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -177,8 +177,7 @@ struct prof_tdata_s { /* Sampling state. */ uint64_t prng_state; - uint64_t threshold; - uint64_t accum; + uint64_t bytes_until_sample; /* State used to avoid dumping while operating on prof internals. */ bool enq; @@ -239,6 +238,7 @@ bool prof_boot2(void); void prof_prefork(void); void prof_postfork_parent(void); void prof_postfork_child(void); +void prof_sample_threshold_update(prof_tdata_t *prof_tdata); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -250,49 +250,13 @@ void prof_postfork_child(void); \ assert(size == s2u(size)); \ \ - prof_tdata = prof_tdata_get(true); \ - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { \ - if (prof_tdata != NULL) \ - ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ - else \ - ret = NULL; \ - break; \ - } \ - \ - if (opt_prof_active == false) { \ - /* Sampling is currently inactive, so avoid sampling. */\ + if (!opt_prof_active || \ + prof_sample_accum_update(size, false, &prof_tdata)) { \ ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ - } else if (opt_lg_prof_sample == 0) { \ - /* Don't bother with sampling logic, since sampling */\ - /* interval is 1. */\ + } else { \ bt_init(&bt, prof_tdata->vec); \ prof_backtrace(&bt, nignore); \ ret = prof_lookup(&bt); \ - } else { \ - if (prof_tdata->threshold == 0) { \ - /* Initialize. Seed the prng differently for */\ - /* each thread. */\ - prof_tdata->prng_state = \ - (uint64_t)(uintptr_t)&size; \ - prof_sample_threshold_update(prof_tdata); \ - } \ - \ - /* Determine whether to capture a backtrace based on */\ - /* whether size is enough for prof_accum to reach */\ - /* prof_tdata->threshold. However, delay updating */\ - /* these variables until prof_{m,re}alloc(), because */\ - /* we don't know for sure that the allocation will */\ - /* succeed. */\ - /* */\ - /* Use subtraction rather than addition to avoid */\ - /* potential integer overflow. */\ - if (size >= prof_tdata->threshold - \ - prof_tdata->accum) { \ - bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore); \ - ret = prof_lookup(&bt); \ - } else \ - ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ } \ } while (0) @@ -300,10 +264,13 @@ void prof_postfork_child(void); malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) prof_tdata_t *prof_tdata_get(bool create); -void prof_sample_threshold_update(prof_tdata_t *prof_tdata); +void prof_sample_accum_update(size_t size, bool commit, + prof_tdata_t **prof_tdata_out); prof_ctx_t *prof_ctx_get(const void *ptr); void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); bool prof_sample_accum_update(size_t size); +void prof_malloc_record_object(const void *ptr, size_t usize, + prof_thr_cnt_t *cnt) void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, size_t old_usize, prof_ctx_t *old_ctx); @@ -330,55 +297,6 @@ prof_tdata_get(bool create) return (prof_tdata); } -JEMALLOC_INLINE void -prof_sample_threshold_update(prof_tdata_t *prof_tdata) -{ - /* - * The body of this function is compiled out unless heap profiling is - * enabled, so that it is possible to compile jemalloc with floating - * point support completely disabled. Avoiding floating point code is - * important on memory-constrained systems, but it also enables a - * workaround for versions of glibc that don't properly save/restore - * floating point registers during dynamic lazy symbol loading (which - * internally calls into whatever malloc implementation happens to be - * integrated into the application). Note that some compilers (e.g. - * gcc 4.8) may use floating point registers for fast memory moves, so - * jemalloc must be compiled with such optimizations disabled (e.g. - * -mno-sse) in order for the workaround to be complete. - */ -#ifdef JEMALLOC_PROF - uint64_t r; - double u; - - cassert(config_prof); - - /* - * Compute sample threshold as a geometrically distributed random - * variable with mean (2^opt_lg_prof_sample). - * - * __ __ - * | log(u) | 1 - * prof_tdata->threshold = | -------- |, where p = ------------------- - * | log(1-p) | opt_lg_prof_sample - * 2 - * - * For more information on the math, see: - * - * Non-Uniform Random Variate Generation - * Luc Devroye - * Springer-Verlag, New York, 1986 - * pp 500 - * (http://luc.devroye.org/rnbookindex.html) - */ - prng64(r, 53, prof_tdata->prng_state, - UINT64_C(6364136223846793005), UINT64_C(1442695040888963407)); - u = (double)r * (1.0/9007199254740992.0L); - prof_tdata->threshold = (uint64_t)(log(u) / - log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) - + (uint64_t)1U; -#endif -} - JEMALLOC_INLINE prof_ctx_t * prof_ctx_get(const void *ptr) { @@ -415,34 +333,58 @@ prof_ctx_set(const void *ptr, prof_ctx_t *ctx) } JEMALLOC_INLINE bool -prof_sample_accum_update(size_t size) +prof_sample_accum_update(size_t size, bool commit, + prof_tdata_t **prof_tdata_out) { prof_tdata_t *prof_tdata; cassert(config_prof); - /* Sampling logic is unnecessary if the interval is 1. */ - assert(opt_lg_prof_sample != 0); - prof_tdata = prof_tdata_get(false); + prof_tdata = prof_tdata_get(true); if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + prof_tdata = NULL; + + if (prof_tdata_out != NULL) + *prof_tdata_out = prof_tdata; + + if (prof_tdata == NULL) return (true); - /* Take care to avoid integer overflow. */ - if (size >= prof_tdata->threshold - prof_tdata->accum) { - prof_tdata->accum -= (prof_tdata->threshold - size); - /* Compute new sample threshold. */ - prof_sample_threshold_update(prof_tdata); - while (prof_tdata->accum >= prof_tdata->threshold) { - prof_tdata->accum -= prof_tdata->threshold; - prof_sample_threshold_update(prof_tdata); - } - return (false); - } else { - prof_tdata->accum += size; + if (prof_tdata->bytes_until_sample >= size) { + if (commit) + prof_tdata->bytes_until_sample -= size; return (true); + } else { + /* Compute new sample threshold. */ + if (commit) + prof_sample_threshold_update(prof_tdata); + return (false); } } +JEMALLOC_INLINE void +prof_malloc_record_object(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) { + prof_ctx_set(ptr, cnt->ctx); + + cnt->epoch++; + /*********/ + mb_write(); + /*********/ + cnt->cnts.curobjs++; + cnt->cnts.curbytes += usize; + if (opt_prof_accum) { + cnt->cnts.accumobjs++; + cnt->cnts.accumbytes += usize; + } + /*********/ + mb_write(); + /*********/ + cnt->epoch++; + /*********/ + mb_write(); + /*********/ +} + JEMALLOC_INLINE void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) { @@ -451,40 +393,20 @@ prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) assert(ptr != NULL); assert(usize == isalloc(ptr, true)); - if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(usize)) { - /* - * Don't sample. For malloc()-like allocation, it is - * always possible to tell in advance how large an - * object's usable size will be, so there should never - * be a difference between the usize passed to - * PROF_ALLOC_PREP() and prof_malloc(). - */ - assert((uintptr_t)cnt == (uintptr_t)1U); - } + if (prof_sample_accum_update(usize, true, NULL)) { + /* + * Don't sample. For malloc()-like allocation, it is + * always possible to tell in advance how large an + * object's usable size will be, so there should never + * be a difference between the usize passed to + * PROF_ALLOC_PREP() and prof_malloc(). + */ + assert((uintptr_t)cnt == (uintptr_t)1U); } - if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); - - cnt->epoch++; - /*********/ - mb_write(); - /*********/ - cnt->cnts.curobjs++; - cnt->cnts.curbytes += usize; - if (opt_prof_accum) { - cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += usize; - } - /*********/ - mb_write(); - /*********/ - cnt->epoch++; - /*********/ - mb_write(); - /*********/ - } else + if ((uintptr_t)cnt > (uintptr_t)1U) + prof_malloc_record_object(ptr, usize, cnt); + else prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); } @@ -499,18 +421,16 @@ prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, if (ptr != NULL) { assert(usize == isalloc(ptr, true)); - if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(usize)) { - /* - * Don't sample. The usize passed to - * PROF_ALLOC_PREP() was larger than what - * actually got allocated, so a backtrace was - * captured for this allocation, even though - * its actual usize was insufficient to cross - * the sample threshold. - */ - cnt = (prof_thr_cnt_t *)(uintptr_t)1U; - } + if (prof_sample_accum_update(usize, true, NULL)) { + /* + * Don't sample. The usize passed to + * PROF_ALLOC_PREP() was larger than what + * actually got allocated, so a backtrace was + * captured for this allocation, even though + * its actual usize was insufficient to cross + * the sample threshold. + */ + cnt = (prof_thr_cnt_t *)(uintptr_t)1U; } } diff --git a/src/prof.c b/src/prof.c index 1b1f7a84..82c7f703 100644 --- a/src/prof.c +++ b/src/prof.c @@ -645,6 +645,66 @@ prof_lookup(prof_bt_t *bt) return (ret.p); } + +void +prof_sample_threshold_update(prof_tdata_t *prof_tdata) +{ + /* + * The body of this function is compiled out unless heap profiling is + * enabled, so that it is possible to compile jemalloc with floating + * point support completely disabled. Avoiding floating point code is + * important on memory-constrained systems, but it also enables a + * workaround for versions of glibc that don't properly save/restore + * floating point registers during dynamic lazy symbol loading (which + * internally calls into whatever malloc implementation happens to be + * integrated into the application). Note that some compilers (e.g. + * gcc 4.8) may use floating point registers for fast memory moves, so + * jemalloc must be compiled with such optimizations disabled (e.g. + * -mno-sse) in order for the workaround to be complete. + */ +#ifdef JEMALLOC_PROF + uint64_t r; + double u; + + if (!config_prof) + return; + + if (prof_tdata == NULL) + prof_tdata = prof_tdata_get(false); + + if (opt_lg_prof_sample == 0) { + prof_tdata->bytes_until_sample = 0; + return; + } + + /* + * Compute sample threshold as a geometrically distributed random + * variable with mean (2^opt_lg_prof_sample). + * + * __ __ + * | log(u) | 1 + * prof_tdata->threshold = | -------- |, where p = ------------------- + * | log(1-p) | opt_lg_prof_sample + * 2 + * + * For more information on the math, see: + * + * Non-Uniform Random Variate Generation + * Luc Devroye + * Springer-Verlag, New York, 1986 + * pp 500 + * (http://luc.devroye.org/rnbookindex.html) + */ + prng64(r, 53, prof_tdata->prng_state, + UINT64_C(6364136223846793005), UINT64_C(1442695040888963407)); + u = (double)r * (1.0/9007199254740992.0L); + prof_tdata->bytes_until_sample = (uint64_t)(log(u) / + log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) + + (uint64_t)1U; +#endif +} + + #ifdef JEMALLOC_JET size_t prof_bt_count(void) @@ -1224,9 +1284,8 @@ prof_tdata_init(void) return (NULL); } - prof_tdata->prng_state = 0; - prof_tdata->threshold = 0; - prof_tdata->accum = 0; + prof_tdata->prng_state = (uint64_t)(uintptr_t)prof_tdata; + prof_sample_threshold_update(prof_tdata); prof_tdata->enq = false; prof_tdata->enq_idump = false; From 021136ce4db79f50031a1fd5dd751891888fbc7b Mon Sep 17 00:00:00 2001 From: Ben Maurer Date: Wed, 16 Apr 2014 14:31:24 -0700 Subject: [PATCH 013/352] Create a const array with only a small bin to size map --- include/jemalloc/internal/arena.h | 3 ++- include/jemalloc/internal/jemalloc_internal.h.in | 4 ++-- include/jemalloc/internal/private_symbols.txt | 1 + include/jemalloc/internal/tcache.h | 6 +++--- src/arena.c | 10 +++++++++- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 0e14c2c4..b435d0b0 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -385,6 +385,7 @@ extern ssize_t opt_lg_dirty_mult; * and all accesses are via the SMALL_SIZE2BIN macro. */ extern uint8_t const small_size2bin[]; +extern uint32_t const small_bin2size[]; #define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) extern arena_bin_info_t arena_bin_info[NBINS]; @@ -964,7 +965,7 @@ arena_salloc(const void *ptr, bool demote) assert(arena_mapbits_large_get(chunk, pageind) != 0 || arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)) == binind); - ret = arena_bin_info[binind].reg_size; + ret = small_bin2size[binind]; } return (ret); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 9c79ae00..17d77623 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -602,7 +602,7 @@ s2u(size_t size) { if (size <= SMALL_MAXCLASS) - return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); + return (small_bin2size[SMALL_SIZE2BIN(size)]); if (size <= arena_maxclass) return (PAGE_CEILING(size)); return (CHUNK_CEILING(size)); @@ -645,7 +645,7 @@ sa2u(size_t size, size_t alignment) if (usize <= arena_maxclass && alignment <= PAGE) { if (usize <= SMALL_MAXCLASS) - return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); + return (small_bin2size[SMALL_SIZE2BIN(usize)]); return (PAGE_CEILING(usize)); } else { size_t run_size; diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 032bed4f..12d64dc4 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -346,6 +346,7 @@ rtree_set s2u sa2u set_errno +small_bin2size small_size2bin stats_cactive stats_cactive_add diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 96447f42..098d19ae 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -266,14 +266,14 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) binind = SMALL_SIZE2BIN(size); assert(binind < NBINS); tbin = &tcache->tbins[binind]; - size = arena_bin_info[binind].reg_size; + size = small_bin2size[binind]; ret = tcache_alloc_easy(tbin); if (ret == NULL) { ret = tcache_alloc_small_hard(tcache, tbin, binind); if (ret == NULL) return (NULL); } - assert(tcache_salloc(ret) == arena_bin_info[binind].reg_size); + assert(tcache_salloc(ret) == size); if (zero == false) { if (config_fill) { @@ -296,7 +296,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) if (config_stats) tbin->tstats.nrequests++; if (config_prof) - tcache->prof_accumbytes += arena_bin_info[binind].reg_size; + tcache->prof_accumbytes += size; tcache_event(tcache); return (ret); } diff --git a/src/arena.c b/src/arena.c index d5741000..37487ffa 100644 --- a/src/arena.c +++ b/src/arena.c @@ -7,6 +7,14 @@ ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; arena_bin_info_t arena_bin_info[NBINS]; +JEMALLOC_ALIGNED(CACHELINE) +const uint32_t small_bin2size[NBINS] = { +#define SIZE_CLASS(bin, delta, size) \ + size, + SIZE_CLASSES +#undef SIZE_CLASS +}; + JEMALLOC_ALIGNED(CACHELINE) const uint8_t small_size2bin[] = { #define S2B_8(i) i, @@ -1615,7 +1623,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) binind = SMALL_SIZE2BIN(size); assert(binind < NBINS); bin = &arena->bins[binind]; - size = arena_bin_info[binind].reg_size; + size = small_bin2size[binind]; malloc_mutex_lock(&bin->lock); if ((run = bin->runcur) != NULL && run->nfree > 0) From 0b49403958b68294eee0eca8a0b5195e761cf316 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 16 Apr 2014 16:38:22 -0700 Subject: [PATCH 014/352] Fix debug-only compilation failures. Fix debug-only compilation failures introduced by changes to prof_sample_accum_update() in: 6c39f9e059d0825f4c29d8cec9f318b798912c3c refactor profiling. only use a bytes till next sample variable. --- include/jemalloc/internal/prof.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 27be10c1..d7422538 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -264,13 +264,12 @@ void prof_sample_threshold_update(prof_tdata_t *prof_tdata); malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) prof_tdata_t *prof_tdata_get(bool create); -void prof_sample_accum_update(size_t size, bool commit, +bool prof_sample_accum_update(size_t size, bool commit, prof_tdata_t **prof_tdata_out); prof_ctx_t *prof_ctx_get(const void *ptr); void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -bool prof_sample_accum_update(size_t size); void prof_malloc_record_object(const void *ptr, size_t usize, - prof_thr_cnt_t *cnt) + prof_thr_cnt_t *cnt); void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, size_t old_usize, prof_ctx_t *old_ctx); From 3541a904d6fb949f3f0aea05418ccce7cbd4b705 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 16 Apr 2014 17:14:33 -0700 Subject: [PATCH 015/352] Refactor small_size2bin and small_bin2size. Refactor small_size2bin and small_bin2size to be inline functions rather than directly accessed arrays. --- include/jemalloc/internal/arena.h | 40 ++++++++++++++----- .../jemalloc/internal/jemalloc_internal.h.in | 26 +++++++----- include/jemalloc/internal/private_symbols.txt | 2 + include/jemalloc/internal/tcache.h | 4 +- src/arena.c | 18 ++++----- 5 files changed, 61 insertions(+), 29 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index fbbbb911..605a87e5 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -380,13 +380,17 @@ struct arena_s { extern ssize_t opt_lg_dirty_mult; /* - * small_size2bin is a compact lookup table that rounds request sizes up to + * small_size2bin_tab is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, - * and all accesses are via the SMALL_SIZE2BIN macro. + * and all accesses are via small_size2bin(). */ -extern uint8_t const small_size2bin[]; -extern uint32_t const small_bin2size[]; -#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) +extern uint8_t const small_size2bin_tab[]; +/* + * small_bin2size_tab duplicates information in arena_bin_info, but in a const + * array, for which it is easier for the compiler to optimize repeated + * dereferences. + */ +extern uint32_t const small_bin2size_tab[NBINS]; extern arena_bin_info_t arena_bin_info[NBINS]; @@ -450,6 +454,8 @@ void arena_postfork_child(arena_t *arena); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +size_t small_size2bin(size_t size); +size_t small_bin2size(size_t binind); arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbitsp_read(size_t *mapbitsp); @@ -492,6 +498,22 @@ void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) # ifdef JEMALLOC_ARENA_INLINE_A +JEMALLOC_ALWAYS_INLINE size_t +small_size2bin(size_t size) +{ + + return ((size_t)(small_size2bin_tab[(size-1) >> LG_TINY_MIN])); +} + +JEMALLOC_ALWAYS_INLINE size_t +small_bin2size(size_t binind) +{ + + return ((size_t)(small_bin2size_tab[binind])); +} +# endif /* JEMALLOC_ARENA_INLINE_A */ + +# ifdef JEMALLOC_ARENA_INLINE_B JEMALLOC_ALWAYS_INLINE arena_chunk_map_t * arena_mapp_get(arena_chunk_t *chunk, size_t pageind) { @@ -773,9 +795,9 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) return (binind); } -# endif /* JEMALLOC_ARENA_INLINE_A */ +# endif /* JEMALLOC_ARENA_INLINE_B */ -# ifdef JEMALLOC_ARENA_INLINE_B +# ifdef JEMALLOC_ARENA_INLINE_C JEMALLOC_INLINE size_t arena_bin_index(arena_t *arena, arena_bin_t *bin) { @@ -965,7 +987,7 @@ arena_salloc(const void *ptr, bool demote) assert(arena_mapbits_large_get(chunk, pageind) != 0 || arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)) == binind); - ret = small_bin2size[binind]; + ret = small_bin2size(binind); } return (ret); @@ -1004,7 +1026,7 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) arena_dalloc_large(chunk->arena, chunk, ptr); } } -# endif /* JEMALLOC_ARENA_INLINE_B */ +# endif /* JEMALLOC_ARENA_INLINE_C */ #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 6f11d4b4..d530c3b8 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -499,6 +499,14 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" +/* + * Include arena.h the first time in order to provide inline functions for this + * header's inlines. + */ +#define JEMALLOC_ARENA_INLINE_A +#include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_INLINE_A + #ifndef JEMALLOC_ENABLE_INLINE malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *) @@ -526,7 +534,7 @@ s2u(size_t size) { if (size <= SMALL_MAXCLASS) - return (small_bin2size[SMALL_SIZE2BIN(size)]); + return (small_bin2size(small_size2bin(size))); if (size <= arena_maxclass) return (PAGE_CEILING(size)); return (CHUNK_CEILING(size)); @@ -569,7 +577,7 @@ sa2u(size_t size, size_t alignment) if (usize <= arena_maxclass && alignment <= PAGE) { if (usize <= SMALL_MAXCLASS) - return (small_bin2size[SMALL_SIZE2BIN(usize)]); + return (small_bin2size(small_size2bin(usize))); return (PAGE_CEILING(usize)); } else { size_t run_size; @@ -643,16 +651,16 @@ choose_arena(arena_t *arena) #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/rtree.h" /* - * Include arena.h twice in order to resolve circular dependencies with - * tcache.h. + * Include arena.h the second and third times in order to resolve circular + * dependencies with tcache.h. */ -#define JEMALLOC_ARENA_INLINE_A -#include "jemalloc/internal/arena.h" -#undef JEMALLOC_ARENA_INLINE_A -#include "jemalloc/internal/tcache.h" #define JEMALLOC_ARENA_INLINE_B #include "jemalloc/internal/arena.h" #undef JEMALLOC_ARENA_INLINE_B +#include "jemalloc/internal/tcache.h" +#define JEMALLOC_ARENA_INLINE_C +#include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_INLINE_C #include "jemalloc/internal/hash.h" #include "jemalloc/internal/quarantine.h" @@ -794,7 +802,7 @@ u2rz(size_t usize) size_t ret; if (usize <= SMALL_MAXCLASS) { - size_t binind = SMALL_SIZE2BIN(usize); + size_t binind = small_size2bin(usize); ret = arena_bin_info[binind].redzone_size; } else ret = 0; diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index ff9ed476..ccbb3a90 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -346,7 +346,9 @@ s2u sa2u set_errno small_bin2size +small_bin2size_tab small_size2bin +small_size2bin_tab stats_cactive stats_cactive_add stats_cactive_get diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 06c7c8fc..c0d48b93 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -263,10 +263,10 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) size_t binind; tcache_bin_t *tbin; - binind = SMALL_SIZE2BIN(size); + binind = small_size2bin(size); assert(binind < NBINS); tbin = &tcache->tbins[binind]; - size = small_bin2size[binind]; + size = small_bin2size(binind); ret = tcache_alloc_easy(tbin); if (ret == NULL) { ret = tcache_alloc_small_hard(tcache, tbin, binind); diff --git a/src/arena.c b/src/arena.c index 4256344c..d956be3e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -8,7 +8,7 @@ ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; arena_bin_info_t arena_bin_info[NBINS]; JEMALLOC_ALIGNED(CACHELINE) -const uint32_t small_bin2size[NBINS] = { +const uint32_t small_bin2size_tab[NBINS] = { #define SIZE_CLASS(bin, delta, size) \ size, SIZE_CLASSES @@ -16,7 +16,7 @@ const uint32_t small_bin2size[NBINS] = { }; JEMALLOC_ALIGNED(CACHELINE) -const uint8_t small_size2bin[] = { +const uint8_t small_size2bin_tab[] = { #define S2B_8(i) i, #define S2B_16(i) S2B_8(i) S2B_8(i) #define S2B_32(i) S2B_16(i) S2B_16(i) @@ -1607,7 +1607,7 @@ arena_quarantine_junk_small(void *ptr, size_t usize) assert(opt_quarantine); assert(usize <= SMALL_MAXCLASS); - binind = SMALL_SIZE2BIN(usize); + binind = small_size2bin(usize); bin_info = &arena_bin_info[binind]; arena_redzones_validate(ptr, bin_info, true); } @@ -1620,10 +1620,10 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) arena_run_t *run; size_t binind; - binind = SMALL_SIZE2BIN(size); + binind = small_size2bin(size); assert(binind < NBINS); bin = &arena->bins[binind]; - size = small_bin2size[binind]; + size = small_bin2size(binind); malloc_mutex_lock(&bin->lock); if ((run = bin->runcur) != NULL && run->nfree > 0) @@ -1777,7 +1777,7 @@ arena_prof_promoted(const void *ptr, size_t size) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - binind = SMALL_SIZE2BIN(size); + binind = small_size2bin(size); assert(binind < NBINS); arena_mapbits_large_binind_set(chunk, pageind, binind); @@ -2164,11 +2164,11 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, */ if (oldsize <= arena_maxclass) { if (oldsize <= SMALL_MAXCLASS) { - assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size + assert(arena_bin_info[small_size2bin(oldsize)].reg_size == oldsize); if ((size + extra <= SMALL_MAXCLASS && - SMALL_SIZE2BIN(size + extra) == - SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && + small_size2bin(size + extra) == + small_size2bin(oldsize)) || (size <= oldsize && size + extra >= oldsize)) return (false); } else { From 9d4e13f45a281a2eabe4d3528ab26e5f3903d5a5 Mon Sep 17 00:00:00 2001 From: Lucian Adrian Grijincu Date: Mon, 21 Apr 2014 20:52:35 -0700 Subject: [PATCH 016/352] prof_backtrace: use unw_backtrace unw_backtrace: - does internal per-thread caching - doesn't acquire an internal lock --- .../jemalloc/internal/jemalloc_internal.h.in | 4 +-- src/prof.c | 33 +++++-------------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index d530c3b8..dc77b5a1 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -352,9 +352,9 @@ static const bool config_ivsalloc = # endif # endif # define VARIABLE_ARRAY(type, name, count) \ - type *name = alloca(sizeof(type) * count) + type *name = alloca(sizeof(type) * (count)) #else -# define VARIABLE_ARRAY(type, name, count) type name[count] +# define VARIABLE_ARRAY(type, name, count) type name[(count)] #endif #include "jemalloc/internal/valgrind.h" diff --git a/src/prof.c b/src/prof.c index 82c7f703..11f1267f 100644 --- a/src/prof.c +++ b/src/prof.c @@ -160,36 +160,21 @@ prof_leave(prof_tdata_t *prof_tdata) void prof_backtrace(prof_bt_t *bt, unsigned nignore) { - unw_context_t uc; - unw_cursor_t cursor; - unsigned i; - int err; - cassert(config_prof); assert(bt->len == 0); assert(bt->vec != NULL); - unw_getcontext(&uc); - unw_init_local(&cursor, &uc); + VARIABLE_ARRAY(void *, frames, nignore + PROF_BT_MAX); + int n = unw_backtrace(frames, nignore + PROF_BT_MAX); + if (n <= 0) + return; /* Throw away (nignore+1) stack frames, if that many exist. */ - for (i = 0; i < nignore + 1; i++) { - err = unw_step(&cursor); - if (err <= 0) - return; - } - - /* - * Iterate over stack frames until there are no more, or until no space - * remains in bt. - */ - for (i = 0; i < PROF_BT_MAX; i++) { - unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); - bt->len++; - err = unw_step(&cursor); - if (err <= 0) - break; - } + nignore++; + if (nignore >= n) + return; + memcpy(bt->vec, &frames[nignore], sizeof(frames[0]) * (n - nignore)); + bt->len = n - nignore; } #elif (defined(JEMALLOC_PROF_LIBGCC)) static _Unwind_Reason_Code From 05125b83778a5695c29777acdc662d999d016d32 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 22 Apr 2014 20:48:07 -0700 Subject: [PATCH 017/352] Update libunwind configuration check to look for unw_backtrace(). Update libunwind configuration check to look for unw_backtrace(), which is a newer API not available in older versions of libunwind. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index dc817e1c..eb9ca45c 100644 --- a/configure.ac +++ b/configure.ac @@ -702,7 +702,7 @@ fi, if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then AC_CHECK_HEADERS([libunwind.h], , [enable_prof_libunwind="0"]) if test "x$LUNWIND" = "x-lunwind" ; then - AC_CHECK_LIB([unwind], [backtrace], [LIBS="$LIBS $LUNWIND"], + AC_CHECK_LIB([unwind], [unw_backtrace], [LIBS="$LIBS $LUNWIND"], [enable_prof_libunwind="0"]) else LIBS="$LIBS $LUNWIND" From 6f001059aa33d77a3cb7799002044faf8dd08fc0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 22 Apr 2014 18:41:15 -0700 Subject: [PATCH 018/352] Simplify backtracing. Simplify backtracing to not ignore any frames, and compensate for this in pprof in order to increase flexibility with respect to function-based refactoring even in the presence of non-deterministic inlining. Modify pprof to blacklist all jemalloc allocation entry points including non-standard ones like mallocx(), and ignore all allocator-internal frames. Prior to this change, pprof excluded the specifically blacklisted functions from backtraces, but it left allocator-internal frames intact. --- bin/pprof | 9 ++++ include/jemalloc/internal/prof.h | 7 ++- src/jemalloc.c | 80 +++++++++++--------------------- src/prof.c | 55 +++++++++------------- 4 files changed, 60 insertions(+), 91 deletions(-) diff --git a/bin/pprof b/bin/pprof index a309943c..328138cd 100755 --- a/bin/pprof +++ b/bin/pprof @@ -2811,9 +2811,14 @@ sub RemoveUninterestingFrames { 'free', 'memalign', 'posix_memalign', + 'aligned_alloc', 'pvalloc', 'valloc', 'realloc', + 'mallocx', # jemalloc + 'rallocx', # jemalloc + 'xallocx', # jemalloc + 'dallocx', # jemalloc 'tc_calloc', 'tc_cfree', 'tc_malloc', @@ -2923,6 +2928,10 @@ sub RemoveUninterestingFrames { if (exists($symbols->{$a})) { my $func = $symbols->{$a}->[0]; if ($skip{$func} || ($func =~ m/$skip_regexp/)) { + # Throw away the portion of the backtrace seen so far, under the + # assumption that previous frames were for functions internal to the + # allocator. + @path = (); next; } } diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index d7422538..d82fbc4f 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -63,7 +63,6 @@ struct prof_bt_s { /* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ typedef struct { prof_bt_t *bt; - unsigned nignore; unsigned max; } prof_unwind_data_t; #endif @@ -220,7 +219,7 @@ extern char opt_prof_prefix[ extern uint64_t prof_interval; void bt_init(prof_bt_t *bt, void **vec); -void prof_backtrace(prof_bt_t *bt, unsigned nignore); +void prof_backtrace(prof_bt_t *bt); prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); #ifdef JEMALLOC_JET size_t prof_bt_count(void); @@ -244,7 +243,7 @@ void prof_sample_threshold_update(prof_tdata_t *prof_tdata); /******************************************************************************/ #ifdef JEMALLOC_H_INLINES -#define PROF_ALLOC_PREP(nignore, size, ret) do { \ +#define PROF_ALLOC_PREP(size, ret) do { \ prof_tdata_t *prof_tdata; \ prof_bt_t bt; \ \ @@ -255,7 +254,7 @@ void prof_sample_threshold_update(prof_tdata_t *prof_tdata); ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ } else { \ bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore); \ + prof_backtrace(&bt); \ ret = prof_lookup(&bt); \ } \ } while (0) diff --git a/src/jemalloc.c b/src/jemalloc.c index 36eae722..f1dda758 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -881,10 +881,12 @@ imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) } JEMALLOC_ALWAYS_INLINE_C void * -imalloc_prof(size_t usize, prof_thr_cnt_t *cnt) +imalloc_prof(size_t usize) { void *p; + prof_thr_cnt_t *cnt; + PROF_ALLOC_PREP(usize, cnt); if ((uintptr_t)cnt != (uintptr_t)1U) p = imalloc_prof_sample(usize, cnt); else @@ -896,42 +898,22 @@ imalloc_prof(size_t usize, prof_thr_cnt_t *cnt) return (p); } -/* - * MALLOC_BODY() is a macro rather than a function because its contents are in - * the fast path, but inlining would cause reliability issues when determining - * how many frames to discard from heap profiling backtraces. - */ -#define MALLOC_BODY(ret, size, usize) do { \ - if (malloc_init()) \ - ret = NULL; \ - else { \ - if (config_prof && opt_prof) { \ - prof_thr_cnt_t *cnt; \ - \ - usize = s2u(size); \ - /* \ - * Call PROF_ALLOC_PREP() here rather than in \ - * imalloc_prof() so that imalloc_prof() can be \ - * inlined without introducing uncertainty \ - * about the number of backtrace frames to \ - * ignore. imalloc_prof() is in the fast path \ - * when heap profiling is enabled, so inlining \ - * is critical to performance. (For \ - * consistency all callers of PROF_ALLOC_PREP() \ - * are structured similarly, even though e.g. \ - * realloc() isn't called enough for inlining \ - * to be critical.) \ - */ \ - PROF_ALLOC_PREP(1, usize, cnt); \ - ret = imalloc_prof(usize, cnt); \ - } else { \ - if (config_stats || (config_valgrind && \ - in_valgrind)) \ - usize = s2u(size); \ - ret = imalloc(size); \ - } \ - } \ -} while (0) +JEMALLOC_ALWAYS_INLINE_C void * +imalloc_body(size_t size, size_t *usize) +{ + + if (malloc_init()) + return (NULL); + + if (config_prof && opt_prof) { + *usize = s2u(size); + return (imalloc_prof(*usize)); + } + + if (config_stats || (config_valgrind && in_valgrind)) + *usize = s2u(size); + return (imalloc(size)); +} void * je_malloc(size_t size) @@ -942,8 +924,7 @@ je_malloc(size_t size) if (size == 0) size = 1; - MALLOC_BODY(ret, size, usize); - + ret = imalloc_body(size, &usize); if (ret == NULL) { if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in malloc(): " @@ -998,13 +979,6 @@ imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) } JEMALLOC_ATTR(nonnull(1)) -#ifdef JEMALLOC_PROF -/* - * Avoid any uncertainty as to how many backtrace frames to ignore in - * PROF_ALLOC_PREP(). - */ -JEMALLOC_NOINLINE -#endif static int imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) { @@ -1043,7 +1017,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) if (config_prof && opt_prof) { prof_thr_cnt_t *cnt; - PROF_ALLOC_PREP(2, usize, cnt); + PROF_ALLOC_PREP(usize, cnt); result = imemalign_prof(alignment, usize, cnt); } else result = ipalloc(usize, alignment, false); @@ -1166,7 +1140,7 @@ je_calloc(size_t num, size_t size) prof_thr_cnt_t *cnt; usize = s2u(num_size); - PROF_ALLOC_PREP(1, usize, cnt); + PROF_ALLOC_PREP(usize, cnt); ret = icalloc_prof(usize, cnt); } else { if (config_stats || (config_valgrind && in_valgrind)) @@ -1282,7 +1256,7 @@ je_realloc(void *ptr, size_t size) prof_thr_cnt_t *cnt; usize = s2u(size); - PROF_ALLOC_PREP(1, usize, cnt); + PROF_ALLOC_PREP(usize, cnt); ret = irealloc_prof(ptr, old_usize, usize, cnt); } else { if (config_stats || (config_valgrind && in_valgrind)) @@ -1291,7 +1265,7 @@ je_realloc(void *ptr, size_t size) } } else { /* realloc(NULL, size) is equivalent to malloc(size). */ - MALLOC_BODY(ret, size, usize); + ret = imalloc_body(size, &usize); } if (ret == NULL) { @@ -1475,7 +1449,7 @@ je_mallocx(size_t size, int flags) if (config_prof && opt_prof) { prof_thr_cnt_t *cnt; - PROF_ALLOC_PREP(1, usize, cnt); + PROF_ALLOC_PREP(usize, cnt); p = imallocx_prof(usize, alignment, zero, try_tcache, arena, cnt); } else @@ -1600,7 +1574,7 @@ je_rallocx(void *ptr, size_t size, int flags) usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); assert(usize != 0); - PROF_ALLOC_PREP(1, usize, cnt); + PROF_ALLOC_PREP(usize, cnt); p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, try_tcache_alloc, try_tcache_dalloc, arena, cnt); if (p == NULL) @@ -1733,7 +1707,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) */ size_t max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment); - PROF_ALLOC_PREP(1, max_usize, cnt); + PROF_ALLOC_PREP(max_usize, cnt); usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, max_usize, zero, arena, cnt); } else { diff --git a/src/prof.c b/src/prof.c index 11f1267f..b64386e3 100644 --- a/src/prof.c +++ b/src/prof.c @@ -158,23 +158,18 @@ prof_leave(prof_tdata_t *prof_tdata) #ifdef JEMALLOC_PROF_LIBUNWIND void -prof_backtrace(prof_bt_t *bt, unsigned nignore) +prof_backtrace(prof_bt_t *bt) { + int nframes; + cassert(config_prof); assert(bt->len == 0); assert(bt->vec != NULL); - VARIABLE_ARRAY(void *, frames, nignore + PROF_BT_MAX); - int n = unw_backtrace(frames, nignore + PROF_BT_MAX); - if (n <= 0) + nframes = unw_backtrace(bt->vec, PROF_BT_MAX); + if (nframes <= 0) return; - - /* Throw away (nignore+1) stack frames, if that many exist. */ - nignore++; - if (nignore >= n) - return; - memcpy(bt->vec, &frames[nignore], sizeof(frames[0]) * (n - nignore)); - bt->len = n - nignore; + bt->len = nframes; } #elif (defined(JEMALLOC_PROF_LIBGCC)) static _Unwind_Reason_Code @@ -190,25 +185,25 @@ static _Unwind_Reason_Code prof_unwind_callback(struct _Unwind_Context *context, void *arg) { prof_unwind_data_t *data = (prof_unwind_data_t *)arg; + void *ip; cassert(config_prof); - if (data->nignore > 0) - data->nignore--; - else { - data->bt->vec[data->bt->len] = (void *)_Unwind_GetIP(context); - data->bt->len++; - if (data->bt->len == data->max) - return (_URC_END_OF_STACK); - } + ip = (void *)_Unwind_GetIP(context); + if (ip == NULL) + return (_URC_END_OF_STACK); + data->bt->vec[data->bt->len] = ip; + data->bt->len++; + if (data->bt->len == data->max) + return (_URC_END_OF_STACK); return (_URC_NO_REASON); } void -prof_backtrace(prof_bt_t *bt, unsigned nignore) +prof_backtrace(prof_bt_t *bt) { - prof_unwind_data_t data = {bt, nignore, PROF_BT_MAX}; + prof_unwind_data_t data = {bt, PROF_BT_MAX}; cassert(config_prof); @@ -216,25 +211,22 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore) } #elif (defined(JEMALLOC_PROF_GCC)) void -prof_backtrace(prof_bt_t *bt, unsigned nignore) +prof_backtrace(prof_bt_t *bt) { #define BT_FRAME(i) \ - if ((i) < nignore + PROF_BT_MAX) { \ + if ((i) < PROF_BT_MAX) { \ void *p; \ if (__builtin_frame_address(i) == 0) \ return; \ p = __builtin_return_address(i); \ if (p == NULL) \ return; \ - if (i >= nignore) { \ - bt->vec[(i) - nignore] = p; \ - bt->len = (i) - nignore + 1; \ - } \ + bt->vec[(i)] = p; \ + bt->len = (i) + 1; \ } else \ return; cassert(config_prof); - assert(nignore <= 3); BT_FRAME(0) BT_FRAME(1) @@ -376,16 +368,11 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore) BT_FRAME(125) BT_FRAME(126) BT_FRAME(127) - - /* Extras to compensate for nignore. */ - BT_FRAME(128) - BT_FRAME(129) - BT_FRAME(130) #undef BT_FRAME } #else void -prof_backtrace(prof_bt_t *bt, unsigned nignore) +prof_backtrace(prof_bt_t *bt) { cassert(config_prof); From a344dd01c74a7e385087819046105f689931905d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 1 May 2014 15:51:30 -0700 Subject: [PATCH 019/352] Fix coding sytle nits. --- src/jemalloc.c | 8 ++++---- test/include/test/test.h | 2 +- test/src/test.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index f1dda758..289d7f74 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1346,10 +1346,10 @@ je_valloc(size_t size) * passed an extra argument for the caller return address, which will be * ignored. */ -JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free; -JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc; -JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc; -JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) = +JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; +JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; +JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; +JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = je_memalign; #endif diff --git a/test/include/test/test.h b/test/include/test/test.h index a32ec07c..161fafdf 100644 --- a/test/include/test/test.h +++ b/test/include/test/test.h @@ -323,7 +323,7 @@ void test_skip(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2)); void test_fail(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2)); /* For private use by macros. */ -test_status_t p_test(test_t* t, ...); +test_status_t p_test(test_t *t, ...); void p_test_init(const char *name); void p_test_fini(void); void p_test_fail(const char *prefix, const char *message); diff --git a/test/src/test.c b/test/src/test.c index 528d8583..3acf8454 100644 --- a/test/src/test.c +++ b/test/src/test.c @@ -61,13 +61,13 @@ p_test_fini(void) } test_status_t -p_test(test_t* t, ...) +p_test(test_t *t, ...) { test_status_t ret = test_status_pass; va_list ap; va_start(ap, t); - for (; t != NULL; t = va_arg(ap, test_t*)) { + for (; t != NULL; t = va_arg(ap, test_t *)) { t(); if (test_status > ret) ret = test_status; From 74b1ea5ce09c8455f35da0fbbd41f678708151d8 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Wed, 7 May 2014 17:58:54 -0400 Subject: [PATCH 020/352] fix git handling of newlines on windows By default, git will coerce LF to CRLF when files are checked out on Windows. This causes hard to diagnose errors when compiling with mingw-w64 from Windows rather than cross-compiling. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6313b56c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf From fb7fe50a88ca9bde74e9a401ae17ad3b15bbae28 Mon Sep 17 00:00:00 2001 From: aravind Date: Mon, 5 May 2014 15:16:56 -0700 Subject: [PATCH 021/352] Add support for user-specified chunk allocators/deallocators. Add new mallctl endpoints "arena.chunk.alloc" and "arena.chunk.dealloc" to allow userspace to configure jemalloc's chunk allocator and deallocator on a per-arena basis. --- Makefile.in | 3 +- doc/jemalloc.xml.in | 63 +++++++++++++++++++ include/jemalloc/internal/arena.h | 6 ++ include/jemalloc/internal/chunk.h | 8 ++- include/jemalloc/internal/extent.h | 3 + include/jemalloc/internal/huge.h | 10 +-- .../jemalloc/internal/jemalloc_internal.h.in | 14 +++-- include/jemalloc/internal/private_symbols.txt | 1 + include/jemalloc/jemalloc_protos.h.in | 3 + src/arena.c | 8 ++- src/base.c | 2 +- src/chunk.c | 58 ++++++++++++----- src/ctl.c | 61 +++++++++++++++++- src/huge.c | 25 ++++---- src/jemalloc.c | 2 +- test/integration/chunk.c | 61 ++++++++++++++++++ 16 files changed, 283 insertions(+), 45 deletions(-) create mode 100644 test/integration/chunk.c diff --git a/Makefile.in b/Makefile.in index e411804a..800dd08d 100644 --- a/Makefile.in +++ b/Makefile.in @@ -142,7 +142,8 @@ TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/rallocx.c \ $(srcroot)test/integration/thread_arena.c \ $(srcroot)test/integration/thread_tcache_enabled.c \ - $(srcroot)test/integration/xallocx.c + $(srcroot)test/integration/xallocx.c \ + $(srcroot)test/integration/chunk.c TESTS_STRESS := TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 78e9b3c6..a7c38b55 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1283,6 +1283,69 @@ malloc_conf = "xmalloc:true";]]> + + + arena.<i>.chunk.alloc + (chunk_alloc_t *) + rw + + Get or set the chunk allocation function for arena + <i>. If setting, the chunk deallocation function should + also be set via + arena.<i>.chunk.dealloc to a companion + function that knows how to deallocate the chunks. + + typedef void *(chunk_alloc_t) + size_t size + size_t alignment + bool *zero + unsigned arena_ind + + A chunk allocation function conforms to the chunk_alloc_t + type and upon success returns a pointer to size + bytes of memory on behalf of arena arena_ind such + that the chunk's base address is a multiple of + alignment, as well as setting + *zero to indicate whether the chunk is zeroed. + Upon error the function returns NULL and leaves + *zero unmodified. The + size parameter is always a multiple of the chunk + size. The alignment parameter is always a power + of two at least as large as the chunk size. Zeroing is mandatory if + *zero is true upon function + entry. + + + + + arena.<i>.chunk.dealloc + (chunk_dealloc_t *) + rw + + Get or set the chunk deallocation function for arena + <i>. If setting, the chunk deallocation function must + be capable of deallocating all extant chunks associated with arena + <i>, usually by passing unknown chunks to the deallocation + function that was replaced. In practice, it is feasible to control + allocation for arenas created via arenas.extend such + that all chunks originate from an application-supplied chunk allocator + (by setting custom chunk allocation/deallocation functions just after + arena creation), but the automatically created arenas may have already + created chunks prior to the application having an opportunity to take + over chunk allocation. + + typedef void (chunk_dealloc_t) + void *chunk + size_t size + unsigned arena_ind + + A chunk deallocation function conforms to the + chunk_dealloc_t type and deallocates a + chunk of given size on + behalf of arena arena_ind. + + arenas.narenas diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 605a87e5..d50159b3 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -370,6 +370,12 @@ struct arena_s { */ arena_avail_tree_t runs_avail; + /* + * user-configureable chunk allocation and deallocation functions. + */ + chunk_alloc_t *chunk_alloc; + chunk_dealloc_t *chunk_dealloc; + /* bins is used to store trees of free regions. */ arena_bin_t bins[NBINS]; }; diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 87d8700d..cea0e8ae 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -43,10 +43,12 @@ extern size_t chunk_npages; extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t arena_maxclass; /* Max size class for arenas. */ -void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, - dss_prec_t dss_prec); +void *chunk_alloc(arena_t *arena, size_t size, size_t alignment, bool base, + bool *zero, dss_prec_t dss_prec); +void *chunk_alloc_default(size_t size, size_t alignment, bool *zero, + unsigned arena_ind); void chunk_unmap(void *chunk, size_t size); -void chunk_dealloc(void *chunk, size_t size, bool unmap); +void chunk_dealloc(arena_t *arena, void *chunk, size_t size, bool unmap); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index ba95ca81..000ef6d5 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -24,6 +24,9 @@ struct extent_node_s { /* Total region size. */ size_t size; + /* Arena from which this extent came, if any */ + arena_t *arena; + /* True if zero-filled; used by chunk recycling code. */ bool zeroed; }; diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index a2b9c779..ab8d44a2 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -17,13 +17,15 @@ extern size_t huge_allocated; /* Protects chunk-related data structures. */ extern malloc_mutex_t huge_mtx; -void *huge_malloc(size_t size, bool zero, dss_prec_t dss_prec); -void *huge_palloc(size_t size, size_t alignment, bool zero, +void *huge_malloc(arena_t *arena, size_t size, bool zero, + dss_prec_t dss_prec); +void *huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero, dss_prec_t dss_prec); bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra); -void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec); +void *huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc, + dss_prec_t dss_prec); #ifdef JEMALLOC_JET typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index dc77b5a1..9e779c65 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -702,7 +702,8 @@ imalloct(size_t size, bool try_tcache, arena_t *arena) if (size <= arena_maxclass) return (arena_malloc(arena, size, false, try_tcache)); else - return (huge_malloc(size, false, huge_dss_prec_get(arena))); + return (huge_malloc(arena, size, false, + huge_dss_prec_get(arena))); } JEMALLOC_ALWAYS_INLINE void * @@ -719,7 +720,8 @@ icalloct(size_t size, bool try_tcache, arena_t *arena) if (size <= arena_maxclass) return (arena_malloc(arena, size, true, try_tcache)); else - return (huge_malloc(size, true, huge_dss_prec_get(arena))); + return (huge_malloc(arena, size, true, + huge_dss_prec_get(arena))); } JEMALLOC_ALWAYS_INLINE void * @@ -745,9 +747,11 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, ret = arena_palloc(choose_arena(arena), usize, alignment, zero); } else if (alignment <= chunksize) - ret = huge_malloc(usize, zero, huge_dss_prec_get(arena)); + ret = huge_malloc(arena, usize, zero, + huge_dss_prec_get(arena)); else - ret = huge_palloc(usize, alignment, zero, huge_dss_prec_get(arena)); + ret = huge_palloc(arena, usize, alignment, zero, + huge_dss_prec_get(arena)); } assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); @@ -915,7 +919,7 @@ iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, alignment, zero, try_tcache_alloc, try_tcache_dalloc)); } else { - return (huge_ralloc(ptr, oldsize, size, extra, + return (huge_ralloc(arena, ptr, oldsize, size, extra, alignment, zero, try_tcache_dalloc, huge_dss_prec_get(arena))); } } diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index ccbb3a90..589b56a1 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -104,6 +104,7 @@ buferror choose_arena choose_arena_hard chunk_alloc +chunk_alloc_default chunk_alloc_dss chunk_alloc_mmap chunk_boot diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 59aeee11..8e945fa5 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -44,3 +44,6 @@ JEMALLOC_EXPORT void * @je_@memalign(size_t alignment, size_t size) #ifdef JEMALLOC_OVERRIDE_VALLOC JEMALLOC_EXPORT void * @je_@valloc(size_t size) JEMALLOC_ATTR(malloc); #endif + +typedef void *(chunk_alloc_t)(size_t, size_t, bool *, unsigned); +typedef bool (chunk_dealloc_t)(void *, size_t, unsigned); diff --git a/src/arena.c b/src/arena.c index d956be3e..6db2b630 100644 --- a/src/arena.c +++ b/src/arena.c @@ -570,8 +570,8 @@ arena_chunk_init_hard(arena_t *arena) zero = false; malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, false, - &zero, arena->dss_prec); + chunk = (arena_chunk_t *)chunk_alloc(arena, chunksize, chunksize, + false, &zero, arena->dss_prec); malloc_mutex_lock(&arena->lock); if (chunk == NULL) return (NULL); @@ -668,7 +668,7 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) arena->spare = chunk; malloc_mutex_unlock(&arena->lock); - chunk_dealloc((void *)spare, chunksize, true); + chunk_dealloc(arena, (void *)spare, chunksize, true); malloc_mutex_lock(&arena->lock); if (config_stats) arena->stats.mapped -= chunksize; @@ -2319,6 +2319,8 @@ arena_new(arena_t *arena, unsigned ind) arena->ind = ind; arena->nthreads = 0; + arena->chunk_alloc = chunk_alloc_default; + arena->chunk_dealloc = (chunk_dealloc_t *)chunk_unmap; if (malloc_mutex_init(&arena->lock)) return (true); diff --git a/src/base.c b/src/base.c index 03dcf8f4..e8b312ef 100644 --- a/src/base.c +++ b/src/base.c @@ -32,7 +32,7 @@ base_pages_alloc(size_t minsize) assert(minsize != 0); csize = CHUNK_CEILING(minsize); zero = false; - base_pages = chunk_alloc(csize, chunksize, true, &zero, + base_pages = chunk_alloc(NULL, csize, chunksize, true, &zero, chunk_dss_prec_get()); if (base_pages == NULL) return (true); diff --git a/src/chunk.c b/src/chunk.c index 246324a2..8bb07229 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -104,7 +104,7 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, malloc_mutex_unlock(&chunks_mtx); node = base_node_alloc(); if (node == NULL) { - chunk_dealloc(ret, size, true); + chunk_dealloc(NULL, ret, size, true); return (NULL); } malloc_mutex_lock(&chunks_mtx); @@ -141,8 +141,8 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, * takes advantage of this to avoid demanding zeroed chunks, but taking * advantage of them if they are returned. */ -void * -chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, +static void * +chunk_alloc_core(size_t size, size_t alignment, bool base, bool *zero, dss_prec_t dss_prec) { void *ret; @@ -156,32 +156,56 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, if (have_dss && dss_prec == dss_prec_primary) { if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment, base, zero)) != NULL) - goto label_return; + return (ret); if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) - goto label_return; + return (ret); } /* mmap. */ if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size, alignment, base, zero)) != NULL) - goto label_return; + return (ret); if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) - goto label_return; + return (ret); /* "secondary" dss. */ if (have_dss && dss_prec == dss_prec_secondary) { if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, alignment, base, zero)) != NULL) - goto label_return; + return (ret); if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) - goto label_return; + return (ret); } /* All strategies for allocation failed. */ - ret = NULL; -label_return: + return (NULL); +} + +/* + * Default arena chunk allocation routine in the absence of user-override. + */ +void * +chunk_alloc_default(size_t size, size_t alignment, bool *zero, + unsigned arena_ind) +{ + + return (chunk_alloc_core(size, alignment, false, zero, + arenas[arena_ind]->dss_prec)); +} + +void * +chunk_alloc(arena_t *arena, size_t size, size_t alignment, bool base, + bool *zero, dss_prec_t dss_prec) +{ + void *ret; + + if (arena) + ret = arena->chunk_alloc(size, alignment, zero, arena->ind); + else + ret = chunk_alloc_core(size, alignment, base, zero, dss_prec); + if (ret != NULL) { if (config_ivsalloc && base == false) { if (rtree_set(chunks_rtree, (uintptr_t)ret, 1)) { - chunk_dealloc(ret, size, true); + chunk_dealloc(arena, ret, size, true); return (NULL); } } @@ -312,7 +336,7 @@ chunk_unmap(void *chunk, size_t size) } void -chunk_dealloc(void *chunk, size_t size, bool unmap) +chunk_dealloc(arena_t *arena, void *chunk, size_t size, bool unmap) { assert(chunk != NULL); @@ -329,8 +353,12 @@ chunk_dealloc(void *chunk, size_t size, bool unmap) malloc_mutex_unlock(&chunks_mtx); } - if (unmap) - chunk_unmap(chunk, size); + if (unmap) { + if (arena) + arena->chunk_dealloc(chunk, size, arena->ind); + else + chunk_unmap(chunk, size); + } } bool diff --git a/src/ctl.c b/src/ctl.c index 9ee5de9f..395c32a1 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -113,6 +113,8 @@ CTL_PROTO(opt_prof_accum) CTL_PROTO(arena_i_purge) static void arena_purge(unsigned arena_ind); CTL_PROTO(arena_i_dss) +CTL_PROTO(arena_i_chunk_alloc) +CTL_PROTO(arena_i_chunk_dealloc) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) @@ -251,9 +253,15 @@ static const ctl_named_node_t opt_node[] = { {NAME("prof_accum"), CTL(opt_prof_accum)} }; +static const ctl_named_node_t chunk_node[] = { + {NAME("alloc"), CTL(arena_i_chunk_alloc)}, + {NAME("dealloc"), CTL(arena_i_chunk_dealloc)} +}; + static const ctl_named_node_t arena_i_node[] = { {NAME("purge"), CTL(arena_i_purge)}, - {NAME("dss"), CTL(arena_i_dss)} + {NAME("dss"), CTL(arena_i_dss)}, + {NAME("chunk"), CHILD(named, chunk)}, }; static const ctl_named_node_t super_arena_i_node[] = { {NAME(""), CHILD(named, arena_i)} @@ -1368,6 +1376,57 @@ label_return: return (ret); } +static int +arena_i_chunk_alloc_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + unsigned arena_ind = mib[1]; + arena_t *arena; + + malloc_mutex_lock(&ctl_mtx); + if (arena_ind < narenas_total && (arena = arenas[arena_ind]) != NULL) { + malloc_mutex_lock(&arena->lock); + READ(arena->chunk_alloc, chunk_alloc_t *); + WRITE(arena->chunk_alloc, chunk_alloc_t *); + } else { + ret = EFAULT; + goto label_outer_return; + } + ret = 0; +label_return: + malloc_mutex_unlock(&arena->lock); +label_outer_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +static int +arena_i_chunk_dealloc_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + + int ret; + unsigned arena_ind = mib[1]; + arena_t *arena; + + malloc_mutex_lock(&ctl_mtx); + if (arena_ind < narenas_total && (arena = arenas[arena_ind]) != NULL) { + malloc_mutex_lock(&arena->lock); + READ(arena->chunk_dealloc, chunk_dealloc_t *); + WRITE(arena->chunk_dealloc, chunk_dealloc_t *); + } else { + ret = EFAULT; + goto label_outer_return; + } + ret = 0; +label_return: + malloc_mutex_unlock(&arena->lock); +label_outer_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + static const ctl_named_node_t * arena_i_index(const size_t *mib, size_t miblen, size_t i) { diff --git a/src/huge.c b/src/huge.c index e725fd90..ab05c905 100644 --- a/src/huge.c +++ b/src/huge.c @@ -16,14 +16,15 @@ malloc_mutex_t huge_mtx; static extent_tree_t huge; void * -huge_malloc(size_t size, bool zero, dss_prec_t dss_prec) +huge_malloc(arena_t *arena, size_t size, bool zero, dss_prec_t dss_prec) { - return (huge_palloc(size, chunksize, zero, dss_prec)); + return (huge_palloc(arena, size, chunksize, zero, dss_prec)); } void * -huge_palloc(size_t size, size_t alignment, bool zero, dss_prec_t dss_prec) +huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero, + dss_prec_t dss_prec) { void *ret; size_t csize; @@ -48,7 +49,7 @@ huge_palloc(size_t size, size_t alignment, bool zero, dss_prec_t dss_prec) * it is possible to make correct junk/zero fill decisions below. */ is_zeroed = zero; - ret = chunk_alloc(csize, alignment, false, &is_zeroed, dss_prec); + ret = chunk_alloc(arena, csize, alignment, false, &is_zeroed, dss_prec); if (ret == NULL) { base_node_dealloc(node); return (NULL); @@ -57,6 +58,7 @@ huge_palloc(size_t size, size_t alignment, bool zero, dss_prec_t dss_prec) /* Insert node into huge. */ node->addr = ret; node->size = csize; + node->arena = arena; malloc_mutex_lock(&huge_mtx); extent_tree_ad_insert(&huge, node); @@ -96,8 +98,9 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) } void * -huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec) +huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc, + dss_prec_t dss_prec) { void *ret; size_t copysize; @@ -112,18 +115,18 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * space and copying. */ if (alignment > chunksize) - ret = huge_palloc(size + extra, alignment, zero, dss_prec); + ret = huge_palloc(arena, size + extra, alignment, zero, dss_prec); else - ret = huge_malloc(size + extra, zero, dss_prec); + ret = huge_malloc(arena, size + extra, zero, dss_prec); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment > chunksize) - ret = huge_palloc(size, alignment, zero, dss_prec); + ret = huge_palloc(arena, size, alignment, zero, dss_prec); else - ret = huge_malloc(size, zero, dss_prec); + ret = huge_malloc(arena, size, zero, dss_prec); if (ret == NULL) return (NULL); @@ -238,7 +241,7 @@ huge_dalloc(void *ptr, bool unmap) if (unmap) huge_dalloc_junk(node->addr, node->size); - chunk_dealloc(node->addr, node->size, unmap); + chunk_dealloc(node->arena, node->addr, node->size, unmap); base_node_dealloc(node); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 289d7f74..e0f9275f 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1983,7 +1983,7 @@ a0alloc(size_t size, bool zero) if (size <= arena_maxclass) return (arena_malloc(arenas[0], size, zero, false)); else - return (huge_malloc(size, zero, huge_dss_prec_get(arenas[0]))); + return (huge_malloc(NULL, size, zero, huge_dss_prec_get(arenas[0]))); } void * diff --git a/test/integration/chunk.c b/test/integration/chunk.c new file mode 100644 index 00000000..13659894 --- /dev/null +++ b/test/integration/chunk.c @@ -0,0 +1,61 @@ +#include "test/jemalloc_test.h" + +chunk_alloc_t *old_alloc; +chunk_dealloc_t *old_dealloc; + +bool +chunk_dealloc(void *chunk, size_t size, unsigned arena_ind) +{ + + return (old_dealloc(chunk, size, arena_ind)); +} + +void * +chunk_alloc(size_t size, size_t alignment, bool *zero, unsigned arena_ind) +{ + + return (old_alloc(size, alignment, zero, arena_ind)); +} + +TEST_BEGIN(test_chunk) +{ + void *p; + chunk_alloc_t *new_alloc; + chunk_dealloc_t *new_dealloc; + size_t old_size, new_size; + + new_alloc = chunk_alloc; + new_dealloc = chunk_dealloc; + old_size = sizeof(chunk_alloc_t *); + new_size = sizeof(chunk_alloc_t *); + + assert_d_eq(mallctl("arena.0.chunk.alloc", &old_alloc, + &old_size, &new_alloc, new_size), 0, + "Unexpected alloc error"); + assert_ptr_ne(old_alloc, new_alloc, + "Unexpected alloc error"); + assert_d_eq(mallctl("arena.0.chunk.dealloc", &old_dealloc, + &old_size, &new_dealloc, new_size), 0, + "Unexpected dealloc error"); + assert_ptr_ne(old_dealloc, new_dealloc, + "Unexpected dealloc error"); + + p = mallocx(42, 0); + assert_ptr_ne(p, NULL, "Unexpected alloc error"); + free(p); + + assert_d_eq(mallctl("arena.0.chunk.alloc", NULL, + NULL, &old_alloc, old_size), 0, + "Unexpected alloc error"); + assert_d_eq(mallctl("arena.0.chunk.dealloc", NULL, + NULL, &old_dealloc, old_size), 0, + "Unexpected dealloc error"); +} +TEST_END + +int +main(void) +{ + + return (test(test_chunk)); +} From e2deab7a751c8080c2b2cdcfd7b11887332be1bb Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 15 May 2014 22:22:27 -0700 Subject: [PATCH 022/352] Refactor huge allocation to be managed by arenas. Refactor huge allocation to be managed by arenas (though the global red-black tree of huge allocations remains for lookup during deallocation). This is the logical conclusion of recent changes that 1) made per arena dss precedence apply to huge allocation, and 2) made it possible to replace the per arena chunk allocation/deallocation functions. Remove the top level huge stats, and replace them with per arena huge stats. Normalize function names and types to *dalloc* (some were *dealloc*). Remove the --enable-mremap option. As jemalloc currently operates, this is a performace regression for some applications, but planned work to logarithmically space huge size classes should provide similar amortized performance. The motivation for this change was that mremap-based huge reallocation forced leaky abstractions that prevented refactoring. --- INSTALL | 6 - Makefile.in | 1 - configure.ac | 28 ---- doc/jemalloc.xml.in | 128 +++++++-------- include/jemalloc/internal/arena.h | 7 +- include/jemalloc/internal/base.h | 2 +- include/jemalloc/internal/chunk.h | 8 +- include/jemalloc/internal/chunk_mmap.h | 2 +- include/jemalloc/internal/ctl.h | 5 - include/jemalloc/internal/huge.h | 20 +-- .../jemalloc/internal/jemalloc_internal.h.in | 23 +-- .../internal/jemalloc_internal_defs.h.in | 7 - include/jemalloc/internal/private_symbols.txt | 13 +- include/jemalloc/internal/stats.h | 5 + include/jemalloc/jemalloc_protos.h.in | 2 +- src/arena.c | 113 +++++++++++-- src/base.c | 12 +- src/chunk.c | 153 ++++++++++-------- src/chunk_mmap.c | 2 +- src/ctl.c | 68 ++++---- src/huge.c | 120 +++----------- src/jemalloc.c | 4 +- src/stats.c | 29 ++-- test/integration/chunk.c | 23 ++- test/integration/mremap.c | 45 ------ test/unit/junk.c | 9 +- test/unit/mallctl.c | 1 - test/unit/stats.c | 18 ++- 28 files changed, 384 insertions(+), 470 deletions(-) delete mode 100644 test/integration/mremap.c diff --git a/INSTALL b/INSTALL index 07f51d1e..2df667ca 100644 --- a/INSTALL +++ b/INSTALL @@ -132,12 +132,6 @@ any of the following arguments (not a definitive list) to 'configure': released in bulk, thus reducing the total number of mutex operations. See the "opt.tcache" option for usage details. ---enable-mremap - Enable huge realloc() via mremap(2). mremap() is disabled by default - because the flavor used is specific to Linux, which has a quirk in its - virtual memory allocation algorithm that causes semi-permanent VM map holes - under normal jemalloc operation. - --disable-munmap Disable virtual memory deallocation via munmap(2); instead keep track of the virtual memory for later use. munmap() is disabled by default (i.e. diff --git a/Makefile.in b/Makefile.in index 800dd08d..90869eb8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -137,7 +137,6 @@ TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/allocated.c \ $(srcroot)test/integration/mallocx.c \ $(srcroot)test/integration/MALLOCX_ARENA.c \ - $(srcroot)test/integration/mremap.c \ $(srcroot)test/integration/posix_memalign.c \ $(srcroot)test/integration/rallocx.c \ $(srcroot)test/integration/thread_arena.c \ diff --git a/configure.ac b/configure.ac index eb9ca45c..57015d1d 100644 --- a/configure.ac +++ b/configure.ac @@ -793,33 +793,6 @@ if test "x$enable_tcache" = "x1" ; then fi AC_SUBST([enable_tcache]) -dnl Disable mremap() for huge realloc() by default. -AC_ARG_ENABLE([mremap], - [AS_HELP_STRING([--enable-mremap], [Enable mremap(2) for huge realloc()])], -[if test "x$enable_mremap" = "xno" ; then - enable_mremap="0" -else - enable_mremap="1" -fi -], -[enable_mremap="0"] -) -if test "x$enable_mremap" = "x1" ; then - JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [ -#define _GNU_SOURCE -#include -], [ -void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); -], [je_cv_mremap_fixed]) - if test "x${je_cv_mremap_fixed}" = "xno" ; then - enable_mremap="0" - fi -fi -if test "x$enable_mremap" = "x1" ; then - AC_DEFINE([JEMALLOC_MREMAP], [ ]) -fi -AC_SUBST([enable_mremap]) - dnl Enable VM deallocation via munmap() by default. AC_ARG_ENABLE([munmap], [AS_HELP_STRING([--disable-munmap], [Disable VM deallocation via munmap(2)])], @@ -1447,7 +1420,6 @@ AC_MSG_RESULT([fill : ${enable_fill}]) AC_MSG_RESULT([utrace : ${enable_utrace}]) AC_MSG_RESULT([valgrind : ${enable_valgrind}]) AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) -AC_MSG_RESULT([mremap : ${enable_mremap}]) AC_MSG_RESULT([munmap : ${enable_munmap}]) AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) AC_MSG_RESULT([tls : ${enable_tls}]) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index a7c38b55..46e505fc 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -486,10 +486,11 @@ for (i = 0; i < nbins; i++) { User objects are broken into three categories according to size: small, large, and huge. Small objects are smaller than one page. Large objects are smaller than the chunk size. Huge objects are a multiple of - the chunk size. Small and large objects are managed by arenas; huge - objects are managed separately in a single data structure that is shared by - all threads. Huge objects are used by applications infrequently enough - that this single data structure is not a scalability issue. + the chunk size. Small and large objects are managed entirely by arenas; + huge objects are additionally aggregated in a single data structure that is + shared by all threads. Huge objects are typically used by applications + infrequently enough that this single data structure is not a scalability + issue. Each chunk that is managed by an arena tracks its contents as runs of contiguous pages (unused, backing a set of small objects, or backing one @@ -647,16 +648,6 @@ for (i = 0; i < nbins; i++) { during build configuration. - - - config.mremap - (bool) - r- - - was specified during - build configuration. - - config.munmap @@ -1273,14 +1264,9 @@ malloc_conf = "xmalloc:true";]]> Set the precedence of dss allocation as related to mmap allocation for arena <i>, or for all arenas if <i> equals arenas.narenas. Note - that even during huge allocation this setting is read from the arena - that would be chosen for small or large allocation so that applications - can depend on consistent dss versus mmap allocation regardless of - allocation size. See opt.dss for supported - settings. - + linkend="arenas.narenas">arenas.narenas. See + opt.dss for supported + settings. @@ -1291,8 +1277,8 @@ malloc_conf = "xmalloc:true";]]> Get or set the chunk allocation function for arena <i>. If setting, the chunk deallocation function should - also be set via - arena.<i>.chunk.dealloc to a companion + also be set via + arena.<i>.chunk.dalloc to a companion function that knows how to deallocate the chunks. typedef void *(chunk_alloc_t) @@ -1313,13 +1299,18 @@ malloc_conf = "xmalloc:true";]]> size. The alignment parameter is always a power of two at least as large as the chunk size. Zeroing is mandatory if *zero is true upon function - entry. + entry. + + Note that replacing the default chunk allocation function makes + the arena's arena.<i>.dss + setting irrelevant. - + - arena.<i>.chunk.dealloc - (chunk_dealloc_t *) + arena.<i>.chunk.dalloc + (chunk_dalloc_t *) rw Get or set the chunk deallocation function for arena @@ -1335,13 +1326,13 @@ malloc_conf = "xmalloc:true";]]> created chunks prior to the application having an opportunity to take over chunk allocation. - typedef void (chunk_dealloc_t) + typedef void (chunk_dalloc_t) void *chunk size_t size unsigned arena_ind A chunk deallocation function conforms to the - chunk_dealloc_t type and deallocates a + chunk_dalloc_t type and deallocates a chunk of given size on behalf of arena arena_ind. @@ -1608,39 +1599,6 @@ malloc_conf = "xmalloc:true";]]> - - - stats.huge.allocated - (size_t) - r- - [] - - Number of bytes currently allocated by huge objects. - - - - - - stats.huge.nmalloc - (uint64_t) - r- - [] - - Cumulative number of huge allocation requests. - - - - - - stats.huge.ndalloc - (uint64_t) - r- - [] - - Cumulative number of huge deallocation requests. - - - stats.arenas.<i>.dss @@ -1817,6 +1775,50 @@ malloc_conf = "xmalloc:true";]]> + + + stats.arenas.<i>.huge.allocated + (size_t) + r- + [] + + Number of bytes currently allocated by huge objects. + + + + + + stats.arenas.<i>.huge.nmalloc + (uint64_t) + r- + [] + + Cumulative number of huge allocation requests served + directly by the arena. + + + + + stats.arenas.<i>.huge.ndalloc + (uint64_t) + r- + [] + + Cumulative number of huge deallocation requests served + directly by the arena. + + + + + stats.arenas.<i>.huge.nrequests + (uint64_t) + r- + [] + + Cumulative number of huge allocation requests. + + + stats.arenas.<i>.bins.<j>.allocated diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index d50159b3..598a89b0 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -345,7 +345,7 @@ struct arena_s { */ arena_chunk_t *spare; - /* Number of pages in active runs. */ + /* Number of pages in active runs and huge regions. */ size_t nactive; /* @@ -374,7 +374,7 @@ struct arena_s { * user-configureable chunk allocation and deallocation functions. */ chunk_alloc_t *chunk_alloc; - chunk_dealloc_t *chunk_dealloc; + chunk_dalloc_t *chunk_dalloc; /* bins is used to store trees of free regions. */ arena_bin_t bins[NBINS]; @@ -403,6 +403,9 @@ extern arena_bin_info_t arena_bin_info[NBINS]; /* Number of large size classes. */ #define nlclasses (chunk_npages - map_bias) +void *arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, + bool *zero); +void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size); void arena_purge_all(arena_t *arena); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, uint64_t prof_accumbytes); diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index 9cf75ffb..3fb80b92 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -12,7 +12,7 @@ void *base_alloc(size_t size); void *base_calloc(size_t number, size_t size); extent_node_t *base_node_alloc(void); -void base_node_dealloc(extent_node_t *node); +void base_node_dalloc(extent_node_t *node); bool base_boot(void); void base_prefork(void); void base_postfork_parent(void); diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index cea0e8ae..f3bfbe08 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -43,12 +43,14 @@ extern size_t chunk_npages; extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t arena_maxclass; /* Max size class for arenas. */ -void *chunk_alloc(arena_t *arena, size_t size, size_t alignment, bool base, - bool *zero, dss_prec_t dss_prec); +void *chunk_alloc_base(size_t size); +void *chunk_alloc_arena(chunk_alloc_t *chunk_alloc, + chunk_dalloc_t *chunk_dalloc, unsigned arena_ind, size_t size, + size_t alignment, bool *zero); void *chunk_alloc_default(size_t size, size_t alignment, bool *zero, unsigned arena_ind); void chunk_unmap(void *chunk, size_t size); -void chunk_dealloc(arena_t *arena, void *chunk, size_t size, bool unmap); +bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/include/jemalloc/internal/chunk_mmap.h b/include/jemalloc/internal/chunk_mmap.h index f24abac7..c5d5c6c0 100644 --- a/include/jemalloc/internal/chunk_mmap.h +++ b/include/jemalloc/internal/chunk_mmap.h @@ -12,7 +12,7 @@ bool pages_purge(void *addr, size_t length); void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero); -bool chunk_dealloc_mmap(void *chunk, size_t size); +bool chunk_dalloc_mmap(void *chunk, size_t size); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index 0ffecc5f..2d301bf1 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -57,11 +57,6 @@ struct ctl_stats_s { uint64_t total; /* stats_chunks.nchunks */ size_t high; /* stats_chunks.highchunks */ } chunks; - struct { - size_t allocated; /* huge_allocated */ - uint64_t nmalloc; /* huge_nmalloc */ - uint64_t ndalloc; /* huge_ndalloc */ - } huge; unsigned narenas; ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ }; diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index ab8d44a2..1e545367 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -9,30 +9,18 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -/* Huge allocation statistics. */ -extern uint64_t huge_nmalloc; -extern uint64_t huge_ndalloc; -extern size_t huge_allocated; - -/* Protects chunk-related data structures. */ -extern malloc_mutex_t huge_mtx; - -void *huge_malloc(arena_t *arena, size_t size, bool zero, - dss_prec_t dss_prec); -void *huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero, - dss_prec_t dss_prec); +void *huge_malloc(arena_t *arena, size_t size, bool zero); +void *huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra); void *huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc, - dss_prec_t dss_prec); + size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc); #ifdef JEMALLOC_JET typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; #endif -void huge_dalloc(void *ptr, bool unmap); +void huge_dalloc(void *ptr); size_t huge_salloc(const void *ptr); -dss_prec_t huge_dss_prec_get(arena_t *arena); prof_ctx_t *huge_prof_ctx_get(const void *ptr); void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); bool huge_boot(void); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 9e779c65..c9462e52 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -122,13 +122,6 @@ static const bool config_prof_libunwind = false #endif ; -static const bool config_mremap = -#ifdef JEMALLOC_MREMAP - true -#else - false -#endif - ; static const bool config_munmap = #ifdef JEMALLOC_MUNMAP true @@ -702,8 +695,7 @@ imalloct(size_t size, bool try_tcache, arena_t *arena) if (size <= arena_maxclass) return (arena_malloc(arena, size, false, try_tcache)); else - return (huge_malloc(arena, size, false, - huge_dss_prec_get(arena))); + return (huge_malloc(arena, size, false)); } JEMALLOC_ALWAYS_INLINE void * @@ -720,8 +712,7 @@ icalloct(size_t size, bool try_tcache, arena_t *arena) if (size <= arena_maxclass) return (arena_malloc(arena, size, true, try_tcache)); else - return (huge_malloc(arena, size, true, - huge_dss_prec_get(arena))); + return (huge_malloc(arena, size, true)); } JEMALLOC_ALWAYS_INLINE void * @@ -747,11 +738,9 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, ret = arena_palloc(choose_arena(arena), usize, alignment, zero); } else if (alignment <= chunksize) - ret = huge_malloc(arena, usize, zero, - huge_dss_prec_get(arena)); + ret = huge_malloc(arena, usize, zero); else - ret = huge_palloc(arena, usize, alignment, zero, - huge_dss_prec_get(arena)); + ret = huge_palloc(arena, usize, alignment, zero); } assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); @@ -833,7 +822,7 @@ idalloct(void *ptr, bool try_tcache) if (chunk != ptr) arena_dalloc(chunk, ptr, try_tcache); else - huge_dalloc(ptr, true); + huge_dalloc(ptr); } JEMALLOC_ALWAYS_INLINE void @@ -920,7 +909,7 @@ iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, try_tcache_dalloc)); } else { return (huge_ralloc(arena, ptr, oldsize, size, extra, - alignment, zero, try_tcache_dalloc, huge_dss_prec_get(arena))); + alignment, zero, try_tcache_dalloc)); } } diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index fc959671..09ddd4f3 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -144,13 +144,6 @@ */ #undef JEMALLOC_MUNMAP -/* - * If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). This is - * disabled by default because it is Linux-specific and it will cause virtual - * memory map holes, much like munmap(2) does. - */ -#undef JEMALLOC_MREMAP - /* TLS is used to map arenas and magazine caches to threads. */ #undef JEMALLOC_TLS diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 589b56a1..f6c4fbcc 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -5,6 +5,8 @@ arena_alloc_junk_small arena_bin_index arena_bin_info arena_boot +arena_chunk_alloc_huge +arena_chunk_dalloc_huge arena_dalloc arena_dalloc_bin arena_dalloc_bin_locked @@ -86,7 +88,7 @@ base_alloc base_boot base_calloc base_node_alloc -base_node_dealloc +base_node_dalloc base_postfork_child base_postfork_parent base_prefork @@ -103,13 +105,14 @@ bt_init buferror choose_arena choose_arena_hard -chunk_alloc +chunk_alloc_arena +chunk_alloc_base chunk_alloc_default chunk_alloc_dss chunk_alloc_mmap chunk_boot -chunk_dealloc -chunk_dealloc_mmap +chunk_dalloc_default +chunk_dalloc_mmap chunk_dss_boot chunk_dss_postfork_child chunk_dss_postfork_parent @@ -198,9 +201,7 @@ huge_allocated huge_boot huge_dalloc huge_dalloc_junk -huge_dss_prec_get huge_malloc -huge_mtx huge_ndalloc huge_nmalloc huge_palloc diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index 27f68e36..ce96476a 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -101,6 +101,11 @@ struct arena_stats_s { uint64_t ndalloc_large; uint64_t nrequests_large; + size_t allocated_huge; + uint64_t nmalloc_huge; + uint64_t ndalloc_huge; + uint64_t nrequests_huge; + /* * One element for each possible size class, including sizes that * overlap with bin size classes. This is necessary because ipalloc() diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 8e945fa5..67268c47 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -46,4 +46,4 @@ JEMALLOC_EXPORT void * @je_@valloc(size_t size) JEMALLOC_ATTR(malloc); #endif typedef void *(chunk_alloc_t)(size_t, size_t, bool *, unsigned); -typedef bool (chunk_dealloc_t)(void *, size_t, unsigned); +typedef bool (chunk_dalloc_t)(void *, size_t, unsigned); diff --git a/src/arena.c b/src/arena.c index 6db2b630..f5d7d062 100644 --- a/src/arena.c +++ b/src/arena.c @@ -559,6 +559,65 @@ arena_chunk_init_spare(arena_t *arena) return (chunk); } +static arena_chunk_t * +arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, + bool *zero) +{ + arena_chunk_t *chunk; + chunk_alloc_t *chunk_alloc; + chunk_dalloc_t *chunk_dalloc; + + chunk_alloc = arena->chunk_alloc; + chunk_dalloc = arena->chunk_dalloc; + malloc_mutex_unlock(&arena->lock); + chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, + arena->ind, size, alignment, zero); + malloc_mutex_lock(&arena->lock); + if (config_stats && chunk != NULL) + arena->stats.mapped += chunksize; + + return (chunk); +} + +void * +arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, + bool *zero) +{ + void *ret; + chunk_alloc_t *chunk_alloc; + chunk_dalloc_t *chunk_dalloc; + + malloc_mutex_lock(&arena->lock); + chunk_alloc = arena->chunk_alloc; + chunk_dalloc = arena->chunk_dalloc; + if (config_stats) { + /* Optimistically update stats prior to unlocking. */ + arena->stats.mapped += size; + arena->stats.allocated_huge += size; + arena->stats.nmalloc_huge++; + arena->stats.nrequests_huge++; + } + arena->nactive += (size >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); + + ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, + size, alignment, zero); + if (config_stats) { + if (ret != NULL) + stats_cactive_add(size); + else { + /* Revert optimistic stats updates. */ + malloc_mutex_lock(&arena->lock); + arena->stats.mapped -= size; + arena->stats.allocated_huge -= size; + arena->stats.nmalloc_huge--; + malloc_mutex_unlock(&arena->lock); + } + } + + return (ret); +} + static arena_chunk_t * arena_chunk_init_hard(arena_t *arena) { @@ -569,14 +628,9 @@ arena_chunk_init_hard(arena_t *arena) assert(arena->spare == NULL); zero = false; - malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc(arena, chunksize, chunksize, - false, &zero, arena->dss_prec); - malloc_mutex_lock(&arena->lock); + chunk = arena_chunk_alloc_internal(arena, chunksize, chunksize, &zero); if (chunk == NULL) return (NULL); - if (config_stats) - arena->stats.mapped += chunksize; chunk->arena = arena; @@ -645,7 +699,38 @@ arena_chunk_alloc(arena_t *arena) } static void -arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) +arena_chunk_dalloc_internal(arena_t *arena, arena_chunk_t *chunk) +{ + chunk_dalloc_t *chunk_dalloc; + + chunk_dalloc = arena->chunk_dalloc; + malloc_mutex_unlock(&arena->lock); + chunk_dalloc((void *)chunk, chunksize, arena->ind); + malloc_mutex_lock(&arena->lock); + if (config_stats) + arena->stats.mapped -= chunksize; +} + +void +arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size) +{ + chunk_dalloc_t *chunk_dalloc; + + malloc_mutex_lock(&arena->lock); + chunk_dalloc = arena->chunk_dalloc; + if (config_stats) { + arena->stats.mapped -= size; + arena->stats.allocated_huge -= size; + arena->stats.ndalloc_huge++; + stats_cactive_sub(size); + } + arena->nactive -= (size >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); + chunk_dalloc(chunk, size, arena->ind); +} + +static void +arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) { assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); @@ -667,11 +752,7 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) arena_chunk_t *spare = arena->spare; arena->spare = chunk; - malloc_mutex_unlock(&arena->lock); - chunk_dealloc(arena, (void *)spare, chunksize, true); - malloc_mutex_lock(&arena->lock); - if (config_stats) - arena->stats.mapped -= chunksize; + arena_chunk_dalloc_internal(arena, spare); } else arena->spare = chunk; } @@ -1231,7 +1312,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) if (size == arena_maxclass) { assert(run_ind == map_bias); assert(run_pages == (arena_maxclass >> LG_PAGE)); - arena_chunk_dealloc(arena, chunk); + arena_chunk_dalloc(arena, chunk); } /* @@ -2283,6 +2364,10 @@ arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, astats->nmalloc_large += arena->stats.nmalloc_large; astats->ndalloc_large += arena->stats.ndalloc_large; astats->nrequests_large += arena->stats.nrequests_large; + astats->allocated_huge += arena->stats.allocated_huge; + astats->nmalloc_huge += arena->stats.nmalloc_huge; + astats->ndalloc_huge += arena->stats.ndalloc_huge; + astats->nrequests_huge += arena->stats.nrequests_huge; for (i = 0; i < nlclasses; i++) { lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; @@ -2320,7 +2405,7 @@ arena_new(arena_t *arena, unsigned ind) arena->ind = ind; arena->nthreads = 0; arena->chunk_alloc = chunk_alloc_default; - arena->chunk_dealloc = (chunk_dealloc_t *)chunk_unmap; + arena->chunk_dalloc = chunk_dalloc_default; if (malloc_mutex_init(&arena->lock)) return (true); diff --git a/src/base.c b/src/base.c index e8b312ef..409c7bb7 100644 --- a/src/base.c +++ b/src/base.c @@ -16,24 +16,16 @@ static void *base_next_addr; static void *base_past_addr; /* Addr immediately past base_pages. */ static extent_node_t *base_nodes; -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static bool base_pages_alloc(size_t minsize); - /******************************************************************************/ static bool base_pages_alloc(size_t minsize) { size_t csize; - bool zero; assert(minsize != 0); csize = CHUNK_CEILING(minsize); - zero = false; - base_pages = chunk_alloc(NULL, csize, chunksize, true, &zero, - chunk_dss_prec_get()); + base_pages = chunk_alloc_base(csize); if (base_pages == NULL) return (true); base_next_addr = base_pages; @@ -100,7 +92,7 @@ base_node_alloc(void) } void -base_node_dealloc(extent_node_t *node) +base_node_dalloc(extent_node_t *node) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); diff --git a/src/chunk.c b/src/chunk.c index 8bb07229..38d02868 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -31,13 +31,12 @@ size_t map_bias; size_t arena_maxclass; /* Max size class for arenas. */ /******************************************************************************/ -/* Function prototypes for non-inline static functions. */ +/* + * Function prototypes for static functions that are referenced prior to + * definition. + */ -static void *chunk_recycle(extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, size_t size, size_t alignment, bool base, - bool *zero); -static void chunk_record(extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, void *chunk, size_t size); +static void chunk_dalloc_core(void *chunk, size_t size); /******************************************************************************/ @@ -104,7 +103,7 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, malloc_mutex_unlock(&chunks_mtx); node = base_node_alloc(); if (node == NULL) { - chunk_dealloc(NULL, ret, size, true); + chunk_dalloc_core(ret, size); return (NULL); } malloc_mutex_lock(&chunks_mtx); @@ -119,7 +118,7 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, malloc_mutex_unlock(&chunks_mtx); if (node != NULL) - base_node_dealloc(node); + base_node_dalloc(node); if (*zero) { if (zeroed == false) memset(ret, 0, size); @@ -179,9 +178,73 @@ chunk_alloc_core(size_t size, size_t alignment, bool base, bool *zero, return (NULL); } -/* - * Default arena chunk allocation routine in the absence of user-override. - */ +static bool +chunk_register(void *chunk, size_t size, bool base) +{ + + assert(chunk != NULL); + assert(CHUNK_ADDR2BASE(chunk) == chunk); + + if (config_ivsalloc && base == false) { + if (rtree_set(chunks_rtree, (uintptr_t)chunk, 1)) + return (true); + } + if (config_stats || config_prof) { + bool gdump; + malloc_mutex_lock(&chunks_mtx); + if (config_stats) + stats_chunks.nchunks += (size / chunksize); + stats_chunks.curchunks += (size / chunksize); + if (stats_chunks.curchunks > stats_chunks.highchunks) { + stats_chunks.highchunks = + stats_chunks.curchunks; + if (config_prof) + gdump = true; + } else if (config_prof) + gdump = false; + malloc_mutex_unlock(&chunks_mtx); + if (config_prof && opt_prof && opt_prof_gdump && gdump) + prof_gdump(); + } + if (config_valgrind) + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(chunk, size); + return (false); +} + +void * +chunk_alloc_base(size_t size) +{ + void *ret; + bool zero; + + zero = false; + ret = chunk_alloc_core(size, chunksize, true, &zero, + chunk_dss_prec_get()); + if (ret == NULL) + return (NULL); + if (chunk_register(ret, size, true)) { + chunk_dalloc_core(ret, size); + return (NULL); + } + return (ret); +} + +void * +chunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc, + unsigned arena_ind, size_t size, size_t alignment, bool *zero) +{ + void *ret; + + ret = chunk_alloc(size, alignment, zero, arena_ind); + if (ret != NULL && chunk_register(ret, size, false)) { + chunk_dalloc(ret, size, arena_ind); + ret = NULL; + } + + return (ret); +} + +/* Default arena chunk allocation routine in the absence of user override. */ void * chunk_alloc_default(size_t size, size_t alignment, bool *zero, unsigned arena_ind) @@ -191,48 +254,6 @@ chunk_alloc_default(size_t size, size_t alignment, bool *zero, arenas[arena_ind]->dss_prec)); } -void * -chunk_alloc(arena_t *arena, size_t size, size_t alignment, bool base, - bool *zero, dss_prec_t dss_prec) -{ - void *ret; - - if (arena) - ret = arena->chunk_alloc(size, alignment, zero, arena->ind); - else - ret = chunk_alloc_core(size, alignment, base, zero, dss_prec); - - if (ret != NULL) { - if (config_ivsalloc && base == false) { - if (rtree_set(chunks_rtree, (uintptr_t)ret, 1)) { - chunk_dealloc(arena, ret, size, true); - return (NULL); - } - } - if (config_stats || config_prof) { - bool gdump; - malloc_mutex_lock(&chunks_mtx); - if (config_stats) - stats_chunks.nchunks += (size / chunksize); - stats_chunks.curchunks += (size / chunksize); - if (stats_chunks.curchunks > stats_chunks.highchunks) { - stats_chunks.highchunks = - stats_chunks.curchunks; - if (config_prof) - gdump = true; - } else if (config_prof) - gdump = false; - malloc_mutex_unlock(&chunks_mtx); - if (config_prof && opt_prof && opt_prof_gdump && gdump) - prof_gdump(); - } - if (config_valgrind) - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); - } - assert(CHUNK_ADDR2BASE(ret) == ret); - return (ret); -} - static void chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, size_t size) @@ -316,9 +337,9 @@ label_return: * avoid potential deadlock. */ if (xnode != NULL) - base_node_dealloc(xnode); + base_node_dalloc(xnode); if (xprev != NULL) - base_node_dealloc(xprev); + base_node_dalloc(xprev); } void @@ -331,12 +352,12 @@ chunk_unmap(void *chunk, size_t size) if (have_dss && chunk_in_dss(chunk)) chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size); - else if (chunk_dealloc_mmap(chunk, size)) + else if (chunk_dalloc_mmap(chunk, size)) chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size); } -void -chunk_dealloc(arena_t *arena, void *chunk, size_t size, bool unmap) +static void +chunk_dalloc_core(void *chunk, size_t size) { assert(chunk != NULL); @@ -353,12 +374,16 @@ chunk_dealloc(arena_t *arena, void *chunk, size_t size, bool unmap) malloc_mutex_unlock(&chunks_mtx); } - if (unmap) { - if (arena) - arena->chunk_dealloc(chunk, size, arena->ind); - else - chunk_unmap(chunk, size); - } + chunk_unmap(chunk, size); +} + +/* Default arena chunk deallocation routine in the absence of user override. */ +bool +chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) +{ + + chunk_dalloc_core(chunk, size); + return (false); } bool diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c index 2056d793..f960e068 100644 --- a/src/chunk_mmap.c +++ b/src/chunk_mmap.c @@ -200,7 +200,7 @@ chunk_alloc_mmap(size_t size, size_t alignment, bool *zero) } bool -chunk_dealloc_mmap(void *chunk, size_t size) +chunk_dalloc_mmap(void *chunk, size_t size) { if (config_munmap) diff --git a/src/ctl.c b/src/ctl.c index 395c32a1..a193605d 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -76,7 +76,6 @@ CTL_PROTO(thread_deallocatedp) CTL_PROTO(config_debug) CTL_PROTO(config_fill) CTL_PROTO(config_lazy_lock) -CTL_PROTO(config_mremap) CTL_PROTO(config_munmap) CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) @@ -114,7 +113,7 @@ CTL_PROTO(arena_i_purge) static void arena_purge(unsigned arena_ind); CTL_PROTO(arena_i_dss) CTL_PROTO(arena_i_chunk_alloc) -CTL_PROTO(arena_i_chunk_dealloc) +CTL_PROTO(arena_i_chunk_dalloc) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) @@ -137,9 +136,6 @@ CTL_PROTO(prof_interval) CTL_PROTO(stats_chunks_current) CTL_PROTO(stats_chunks_total) CTL_PROTO(stats_chunks_high) -CTL_PROTO(stats_huge_allocated) -CTL_PROTO(stats_huge_nmalloc) -CTL_PROTO(stats_huge_ndalloc) CTL_PROTO(stats_arenas_i_small_allocated) CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_ndalloc) @@ -148,6 +144,10 @@ CTL_PROTO(stats_arenas_i_large_allocated) CTL_PROTO(stats_arenas_i_large_nmalloc) CTL_PROTO(stats_arenas_i_large_ndalloc) CTL_PROTO(stats_arenas_i_large_nrequests) +CTL_PROTO(stats_arenas_i_huge_allocated) +CTL_PROTO(stats_arenas_i_huge_nmalloc) +CTL_PROTO(stats_arenas_i_huge_ndalloc) +CTL_PROTO(stats_arenas_i_huge_nrequests) CTL_PROTO(stats_arenas_i_bins_j_allocated) CTL_PROTO(stats_arenas_i_bins_j_nmalloc) CTL_PROTO(stats_arenas_i_bins_j_ndalloc) @@ -214,7 +214,6 @@ static const ctl_named_node_t config_node[] = { {NAME("debug"), CTL(config_debug)}, {NAME("fill"), CTL(config_fill)}, {NAME("lazy_lock"), CTL(config_lazy_lock)}, - {NAME("mremap"), CTL(config_mremap)}, {NAME("munmap"), CTL(config_munmap)}, {NAME("prof"), CTL(config_prof)}, {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, @@ -255,7 +254,7 @@ static const ctl_named_node_t opt_node[] = { static const ctl_named_node_t chunk_node[] = { {NAME("alloc"), CTL(arena_i_chunk_alloc)}, - {NAME("dealloc"), CTL(arena_i_chunk_dealloc)} + {NAME("dalloc"), CTL(arena_i_chunk_dalloc)} }; static const ctl_named_node_t arena_i_node[] = { @@ -321,12 +320,6 @@ static const ctl_named_node_t stats_chunks_node[] = { {NAME("high"), CTL(stats_chunks_high)} }; -static const ctl_named_node_t stats_huge_node[] = { - {NAME("allocated"), CTL(stats_huge_allocated)}, - {NAME("nmalloc"), CTL(stats_huge_nmalloc)}, - {NAME("ndalloc"), CTL(stats_huge_ndalloc)} -}; - static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, @@ -341,6 +334,13 @@ static const ctl_named_node_t stats_arenas_i_large_node[] = { {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} }; +static const ctl_named_node_t stats_arenas_i_huge_node[] = { + {NAME("allocated"), CTL(stats_arenas_i_huge_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_huge_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_huge_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_huge_nrequests)}, +}; + static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, @@ -385,6 +385,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("purged"), CTL(stats_arenas_i_purged)}, {NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("large"), CHILD(named, stats_arenas_i_large)}, + {NAME("huge"), CHILD(named, stats_arenas_i_huge)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)} }; @@ -402,7 +403,6 @@ static const ctl_named_node_t stats_node[] = { {NAME("active"), CTL(stats_active)}, {NAME("mapped"), CTL(stats_mapped)}, {NAME("chunks"), CHILD(named, stats_chunks)}, - {NAME("huge"), CHILD(named, stats_huge)}, {NAME("arenas"), CHILD(indexed, stats_arenas)} }; @@ -500,6 +500,11 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->astats.ndalloc_large += astats->astats.ndalloc_large; sstats->astats.nrequests_large += astats->astats.nrequests_large; + sstats->astats.allocated_huge += astats->astats.allocated_huge; + sstats->astats.nmalloc_huge += astats->astats.nmalloc_huge; + sstats->astats.ndalloc_huge += astats->astats.ndalloc_huge; + sstats->astats.nrequests_huge += astats->astats.nrequests_huge; + for (i = 0; i < nlclasses; i++) { sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; @@ -626,12 +631,6 @@ ctl_refresh(void) ctl_stats.chunks.total = stats_chunks.nchunks; ctl_stats.chunks.high = stats_chunks.highchunks; malloc_mutex_unlock(&chunks_mtx); - - malloc_mutex_lock(&huge_mtx); - ctl_stats.huge.allocated = huge_allocated; - ctl_stats.huge.nmalloc = huge_nmalloc; - ctl_stats.huge.ndalloc = huge_ndalloc; - malloc_mutex_unlock(&huge_mtx); } /* @@ -662,10 +661,9 @@ ctl_refresh(void) ctl_stats.allocated = ctl_stats.arenas[ctl_stats.narenas].allocated_small + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large - + ctl_stats.huge.allocated; + + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge; ctl_stats.active = - (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE) - + ctl_stats.huge.allocated; + (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE); ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); } @@ -1140,7 +1138,6 @@ label_return: CTL_RO_BOOL_CONFIG_GEN(config_debug) CTL_RO_BOOL_CONFIG_GEN(config_fill) CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) -CTL_RO_BOOL_CONFIG_GEN(config_mremap) CTL_RO_BOOL_CONFIG_GEN(config_munmap) CTL_RO_BOOL_CONFIG_GEN(config_prof) CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc) @@ -1377,8 +1374,8 @@ label_return: } static int -arena_i_chunk_alloc_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) +arena_i_chunk_alloc_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned arena_ind = mib[1]; @@ -1402,8 +1399,8 @@ label_outer_return: } static int -arena_i_chunk_dealloc_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) +arena_i_chunk_dalloc_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; @@ -1413,8 +1410,8 @@ arena_i_chunk_dealloc_ctl(const size_t *mib, size_t miblen, void *oldp, size_t * malloc_mutex_lock(&ctl_mtx); if (arena_ind < narenas_total && (arena = arenas[arena_ind]) != NULL) { malloc_mutex_lock(&arena->lock); - READ(arena->chunk_dealloc, chunk_dealloc_t *); - WRITE(arena->chunk_dealloc, chunk_dealloc_t *); + READ(arena->chunk_dalloc, chunk_dalloc_t *); + WRITE(arena->chunk_dalloc, chunk_dalloc_t *); } else { ret = EFAULT; goto label_outer_return; @@ -1611,9 +1608,6 @@ CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current, size_t) CTL_RO_CGEN(config_stats, stats_chunks_total, ctl_stats.chunks.total, uint64_t) CTL_RO_CGEN(config_stats, stats_chunks_high, ctl_stats.chunks.high, size_t) -CTL_RO_CGEN(config_stats, stats_huge_allocated, huge_allocated, size_t) -CTL_RO_CGEN(config_stats, stats_huge_nmalloc, huge_nmalloc, uint64_t) -CTL_RO_CGEN(config_stats, stats_huge_ndalloc, huge_ndalloc, uint64_t) CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) @@ -1644,6 +1638,14 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_huge_allocated, + ctl_stats.arenas[mib[2]].astats.allocated_huge, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nmalloc, + ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_huge_ndalloc, + ctl_stats.arenas[mib[2]].astats.ndalloc_huge, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nrequests, + ctl_stats.arenas[mib[2]].astats.nrequests_huge, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_allocated, ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t) diff --git a/src/huge.c b/src/huge.c index ab05c905..d08ed4a9 100644 --- a/src/huge.c +++ b/src/huge.c @@ -4,11 +4,8 @@ /******************************************************************************/ /* Data. */ -uint64_t huge_nmalloc; -uint64_t huge_ndalloc; -size_t huge_allocated; - -malloc_mutex_t huge_mtx; +/* Protects chunk-related data structures. */ +static malloc_mutex_t huge_mtx; /******************************************************************************/ @@ -16,15 +13,14 @@ malloc_mutex_t huge_mtx; static extent_tree_t huge; void * -huge_malloc(arena_t *arena, size_t size, bool zero, dss_prec_t dss_prec) +huge_malloc(arena_t *arena, size_t size, bool zero) { - return (huge_palloc(arena, size, chunksize, zero, dss_prec)); + return (huge_palloc(arena, size, chunksize, zero)); } void * -huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero, - dss_prec_t dss_prec) +huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) { void *ret; size_t csize; @@ -49,9 +45,10 @@ huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero, * it is possible to make correct junk/zero fill decisions below. */ is_zeroed = zero; - ret = chunk_alloc(arena, csize, alignment, false, &is_zeroed, dss_prec); + arena = choose_arena(arena); + ret = arena_chunk_alloc_huge(arena, csize, alignment, &is_zeroed); if (ret == NULL) { - base_node_dealloc(node); + base_node_dalloc(node); return (NULL); } @@ -62,11 +59,6 @@ huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero, malloc_mutex_lock(&huge_mtx); extent_tree_ad_insert(&huge, node); - if (config_stats) { - stats_cactive_add(csize); - huge_nmalloc++; - huge_allocated += csize; - } malloc_mutex_unlock(&huge_mtx); if (config_fill && zero == false) { @@ -99,8 +91,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) void * huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc, - dss_prec_t dss_prec) + size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc) { void *ret; size_t copysize; @@ -115,18 +106,18 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, * space and copying. */ if (alignment > chunksize) - ret = huge_palloc(arena, size + extra, alignment, zero, dss_prec); + ret = huge_palloc(arena, size + extra, alignment, zero); else - ret = huge_malloc(arena, size + extra, zero, dss_prec); + ret = huge_malloc(arena, size + extra, zero); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment > chunksize) - ret = huge_palloc(arena, size, alignment, zero, dss_prec); + ret = huge_palloc(arena, size, alignment, zero); else - ret = huge_malloc(arena, size, zero, dss_prec); + ret = huge_malloc(arena, size, zero); if (ret == NULL) return (NULL); @@ -137,59 +128,8 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, * expectation that the extra bytes will be reliably preserved. */ copysize = (size < oldsize) ? size : oldsize; - -#ifdef JEMALLOC_MREMAP - /* - * Use mremap(2) if this is a huge-->huge reallocation, and neither the - * source nor the destination are in dss. - */ - if (oldsize >= chunksize && (have_dss == false || (chunk_in_dss(ptr) - == false && chunk_in_dss(ret) == false))) { - size_t newsize = huge_salloc(ret); - - /* - * Remove ptr from the tree of huge allocations before - * performing the remap operation, in order to avoid the - * possibility of another thread acquiring that mapping before - * this one removes it from the tree. - */ - huge_dalloc(ptr, false); - if (mremap(ptr, oldsize, newsize, MREMAP_MAYMOVE|MREMAP_FIXED, - ret) == MAP_FAILED) { - /* - * Assuming no chunk management bugs in the allocator, - * the only documented way an error can occur here is - * if the application changed the map type for a - * portion of the old allocation. This is firmly in - * undefined behavior territory, so write a diagnostic - * message, and optionally abort. - */ - char buf[BUFERROR_BUF]; - - buferror(get_errno(), buf, sizeof(buf)); - malloc_printf(": Error in mremap(): %s\n", - buf); - if (opt_abort) - abort(); - memcpy(ret, ptr, copysize); - chunk_dealloc_mmap(ptr, oldsize); - } else if (config_fill && zero == false && opt_junk && oldsize - < newsize) { - /* - * mremap(2) clobbers the original mapping, so - * junk/zero filling is not preserved. There is no - * need to zero fill here, since any trailing - * uninititialized memory is demand-zeroed by the - * kernel, but junk filling must be redone. - */ - memset(ret + oldsize, 0xa5, newsize - oldsize); - } - } else -#endif - { - memcpy(ret, ptr, copysize); - iqalloct(ptr, try_tcache_dalloc); - } + memcpy(ret, ptr, copysize); + iqalloct(ptr, try_tcache_dalloc); return (ret); } @@ -217,7 +157,7 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); #endif void -huge_dalloc(void *ptr, bool unmap) +huge_dalloc(void *ptr) { extent_node_t *node, key; @@ -230,20 +170,11 @@ huge_dalloc(void *ptr, bool unmap) assert(node->addr == ptr); extent_tree_ad_remove(&huge, node); - if (config_stats) { - stats_cactive_sub(node->size); - huge_ndalloc++; - huge_allocated -= node->size; - } - malloc_mutex_unlock(&huge_mtx); - if (unmap) - huge_dalloc_junk(node->addr, node->size); - - chunk_dealloc(node->arena, node->addr, node->size, unmap); - - base_node_dealloc(node); + huge_dalloc_junk(node->addr, node->size); + arena_chunk_dalloc_huge(node->arena, node->addr, node->size); + base_node_dalloc(node); } size_t @@ -266,13 +197,6 @@ huge_salloc(const void *ptr) return (ret); } -dss_prec_t -huge_dss_prec_get(arena_t *arena) -{ - - return (arena_dss_prec_get(choose_arena(arena))); -} - prof_ctx_t * huge_prof_ctx_get(const void *ptr) { @@ -319,12 +243,6 @@ huge_boot(void) return (true); extent_tree_ad_new(&huge); - if (config_stats) { - huge_nmalloc = 0; - huge_ndalloc = 0; - huge_allocated = 0; - } - return (false); } diff --git a/src/jemalloc.c b/src/jemalloc.c index e0f9275f..43a494e4 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1983,7 +1983,7 @@ a0alloc(size_t size, bool zero) if (size <= arena_maxclass) return (arena_malloc(arenas[0], size, zero, false)); else - return (huge_malloc(NULL, size, zero, huge_dss_prec_get(arenas[0]))); + return (huge_malloc(NULL, size, zero)); } void * @@ -2012,7 +2012,7 @@ a0free(void *ptr) if (chunk != ptr) arena_dalloc(chunk, ptr, false); else - huge_dalloc(ptr, true); + huge_dalloc(ptr); } /******************************************************************************/ diff --git a/src/stats.c b/src/stats.c index bef2ab33..a0eb2971 100644 --- a/src/stats.c +++ b/src/stats.c @@ -213,6 +213,8 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, uint64_t small_nmalloc, small_ndalloc, small_nrequests; size_t large_allocated; uint64_t large_nmalloc, large_ndalloc, large_nrequests; + size_t huge_allocated; + uint64_t huge_nmalloc, huge_ndalloc, huge_nrequests; CTL_GET("arenas.page", &page, size_t); @@ -249,12 +251,19 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", large_allocated, large_nmalloc, large_ndalloc, large_nrequests); + CTL_I_GET("stats.arenas.0.huge.allocated", &huge_allocated, size_t); + CTL_I_GET("stats.arenas.0.huge.nmalloc", &huge_nmalloc, uint64_t); + CTL_I_GET("stats.arenas.0.huge.ndalloc", &huge_ndalloc, uint64_t); + CTL_I_GET("stats.arenas.0.huge.nrequests", &huge_nrequests, uint64_t); + malloc_cprintf(write_cb, cbopaque, + "huge: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + huge_allocated, huge_nmalloc, huge_ndalloc, huge_nrequests); malloc_cprintf(write_cb, cbopaque, "total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", - small_allocated + large_allocated, - small_nmalloc + large_nmalloc, - small_ndalloc + large_ndalloc, - small_nrequests + large_nrequests); + small_allocated + large_allocated + huge_allocated, + small_nmalloc + large_nmalloc + huge_nmalloc, + small_ndalloc + large_ndalloc + huge_ndalloc, + small_nrequests + large_nrequests + huge_nrequests); malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", pactive * page); CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); @@ -458,8 +467,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t allocated, active, mapped; size_t chunks_current, chunks_high; uint64_t chunks_total; - size_t huge_allocated; - uint64_t huge_nmalloc, huge_ndalloc; CTL_GET("stats.cactive", &cactive, size_t *); CTL_GET("stats.allocated", &allocated, size_t); @@ -481,16 +488,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, " %13"PRIu64" %12zu %12zu\n", chunks_total, chunks_high, chunks_current); - /* Print huge stats. */ - CTL_GET("stats.huge.nmalloc", &huge_nmalloc, uint64_t); - CTL_GET("stats.huge.ndalloc", &huge_ndalloc, uint64_t); - CTL_GET("stats.huge.allocated", &huge_allocated, size_t); - malloc_cprintf(write_cb, cbopaque, - "huge: nmalloc ndalloc allocated\n"); - malloc_cprintf(write_cb, cbopaque, - " %12"PRIu64" %12"PRIu64" %12zu\n", - huge_nmalloc, huge_ndalloc, huge_allocated); - if (merged) { unsigned narenas; diff --git a/test/integration/chunk.c b/test/integration/chunk.c index 13659894..28537098 100644 --- a/test/integration/chunk.c +++ b/test/integration/chunk.c @@ -1,13 +1,13 @@ #include "test/jemalloc_test.h" chunk_alloc_t *old_alloc; -chunk_dealloc_t *old_dealloc; +chunk_dalloc_t *old_dalloc; bool -chunk_dealloc(void *chunk, size_t size, unsigned arena_ind) +chunk_dalloc(void *chunk, size_t size, unsigned arena_ind) { - return (old_dealloc(chunk, size, arena_ind)); + return (old_dalloc(chunk, size, arena_ind)); } void * @@ -21,11 +21,11 @@ TEST_BEGIN(test_chunk) { void *p; chunk_alloc_t *new_alloc; - chunk_dealloc_t *new_dealloc; + chunk_dalloc_t *new_dalloc; size_t old_size, new_size; new_alloc = chunk_alloc; - new_dealloc = chunk_dealloc; + new_dalloc = chunk_dalloc; old_size = sizeof(chunk_alloc_t *); new_size = sizeof(chunk_alloc_t *); @@ -34,11 +34,9 @@ TEST_BEGIN(test_chunk) "Unexpected alloc error"); assert_ptr_ne(old_alloc, new_alloc, "Unexpected alloc error"); - assert_d_eq(mallctl("arena.0.chunk.dealloc", &old_dealloc, - &old_size, &new_dealloc, new_size), 0, - "Unexpected dealloc error"); - assert_ptr_ne(old_dealloc, new_dealloc, - "Unexpected dealloc error"); + assert_d_eq(mallctl("arena.0.chunk.dalloc", &old_dalloc, &old_size, + &new_dalloc, new_size), 0, "Unexpected dalloc error"); + assert_ptr_ne(old_dalloc, new_dalloc, "Unexpected dalloc error"); p = mallocx(42, 0); assert_ptr_ne(p, NULL, "Unexpected alloc error"); @@ -47,9 +45,8 @@ TEST_BEGIN(test_chunk) assert_d_eq(mallctl("arena.0.chunk.alloc", NULL, NULL, &old_alloc, old_size), 0, "Unexpected alloc error"); - assert_d_eq(mallctl("arena.0.chunk.dealloc", NULL, - NULL, &old_dealloc, old_size), 0, - "Unexpected dealloc error"); + assert_d_eq(mallctl("arena.0.chunk.dalloc", NULL, NULL, &old_dalloc, + old_size), 0, "Unexpected dalloc error"); } TEST_END diff --git a/test/integration/mremap.c b/test/integration/mremap.c deleted file mode 100644 index a7fb7ef0..00000000 --- a/test/integration/mremap.c +++ /dev/null @@ -1,45 +0,0 @@ -#include "test/jemalloc_test.h" - -TEST_BEGIN(test_mremap) -{ - int err; - size_t sz, lg_chunk, chunksize, i; - char *p, *q; - - sz = sizeof(lg_chunk); - err = mallctl("opt.lg_chunk", &lg_chunk, &sz, NULL, 0); - assert_d_eq(err, 0, "Error in mallctl(): %s", strerror(err)); - chunksize = ((size_t)1U) << lg_chunk; - - p = (char *)malloc(chunksize); - assert_ptr_not_null(p, "malloc(%zu) --> %p", chunksize, p); - memset(p, 'a', chunksize); - - q = (char *)realloc(p, chunksize * 2); - assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize * 2, - q); - for (i = 0; i < chunksize; i++) { - assert_c_eq(q[i], 'a', - "realloc() should preserve existing bytes across copies"); - } - - p = q; - - q = (char *)realloc(p, chunksize); - assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize, q); - for (i = 0; i < chunksize; i++) { - assert_c_eq(q[i], 'a', - "realloc() should preserve existing bytes across copies"); - } - - free(q); -} -TEST_END - -int -main(void) -{ - - return (test( - test_mremap)); -} diff --git a/test/unit/junk.c b/test/unit/junk.c index 85bbf9e2..301428f2 100644 --- a/test/unit/junk.c +++ b/test/unit/junk.c @@ -92,12 +92,9 @@ test_junk(size_t sz_min, size_t sz_max) s = (char *)rallocx(s, sz+1, 0); assert_ptr_not_null((void *)s, "Unexpected rallocx() failure"); - if (!config_mremap || sz+1 <= arena_maxclass) { - assert_ptr_eq(most_recently_junked, junked, - "Expected region of size %zu to be " - "junk-filled", - sz); - } + assert_ptr_eq(most_recently_junked, junked, + "Expected region of size %zu to be junk-filled", + sz); } } diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 754834c1..cb120497 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -129,7 +129,6 @@ TEST_BEGIN(test_mallctl_config) TEST_MALLCTL_CONFIG(debug); TEST_MALLCTL_CONFIG(fill); TEST_MALLCTL_CONFIG(lazy_lock); - TEST_MALLCTL_CONFIG(mremap); TEST_MALLCTL_CONFIG(munmap); TEST_MALLCTL_CONFIG(prof); TEST_MALLCTL_CONFIG(prof_libgcc); diff --git a/test/unit/stats.c b/test/unit/stats.c index 03a55c7f..ab87b29b 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -60,7 +60,7 @@ TEST_BEGIN(test_stats_huge) void *p; uint64_t epoch; size_t allocated; - uint64_t nmalloc, ndalloc; + uint64_t nmalloc, ndalloc, nrequests; size_t sz; int expected = config_stats ? 0 : ENOENT; @@ -71,19 +71,23 @@ TEST_BEGIN(test_stats_huge) "Unexpected mallctl() failure"); sz = sizeof(size_t); - assert_d_eq(mallctl("stats.huge.allocated", &allocated, &sz, NULL, 0), - expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); sz = sizeof(uint64_t); - assert_d_eq(mallctl("stats.huge.nmalloc", &nmalloc, &sz, NULL, 0), - expected, "Unexpected mallctl() result"); - assert_d_eq(mallctl("stats.huge.ndalloc", &ndalloc, &sz, NULL, 0), - expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz, NULL, + 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz, NULL, + 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.huge.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); if (config_stats) { assert_zu_gt(allocated, 0, "allocated should be greater than zero"); assert_u64_ge(nmalloc, ndalloc, "nmalloc should be at least as large as ndalloc"); + assert_u64_le(nmalloc, nrequests, + "nmalloc should no larger than nrequests"); } dallocx(p, 0); From b4d62cd61b46130b7947c3a427a2b007e7fa0eb8 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 15 May 2014 22:46:24 -0700 Subject: [PATCH 023/352] Minor doc edit. --- doc/jemalloc.xml.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 46e505fc..308d0c65 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -189,15 +189,15 @@ The posix_memalign function allocates size bytes of memory such that the - allocation's base address is an even multiple of + allocation's base address is a multiple of alignment, and returns the allocation in the value pointed to by ptr. The requested - alignment must be a power of 2 at least as large - as sizeof(void *). + alignment must be a power of 2 at least as large as + sizeof(void *). The aligned_alloc function allocates size bytes of memory such that the - allocation's base address is an even multiple of + allocation's base address is a multiple of alignment. The requested alignment must be a power of 2. Behavior is undefined if size is not an integral multiple of From ed0b0ec935a6df9ef429e56a08c0c9b63c3ba358 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 16:38:24 +0900 Subject: [PATCH 024/352] Fix manual dependency on jemalloc_test.h --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 90869eb8..4cb1a65a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -228,7 +228,7 @@ HEADER_DIRS = $(srcroot)include/jemalloc/internal \ $(objroot)include/jemalloc $(objroot)include/jemalloc/internal HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h)) $(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): $(HEADERS) -$(TESTS_OBJS): $(objroot)test/unit/jemalloc_test.h +$(TESTS_OBJS): $(objroot)test/include/test/jemalloc_test.h endif $(C_OBJS) $(C_PIC_OBJS) $(C_JET_OBJS) $(C_TESTLIB_OBJS) $(TESTS_OBJS): %.$(O): From 47d58a01ff9d894f854412f3f6d3ba97a7aa2929 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 16:59:50 +0900 Subject: [PATCH 025/352] Define _CRT_SPINCOUNT in test/src/mtx.c like in src/mutex.c --- test/src/mtx.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/src/mtx.c b/test/src/mtx.c index 41b95d59..73bd02f6 100644 --- a/test/src/mtx.c +++ b/test/src/mtx.c @@ -1,5 +1,9 @@ #include "test/jemalloc_test.h" +#ifndef _CRT_SPINCOUNT +#define _CRT_SPINCOUNT 4000 +#endif + bool mtx_init(mtx_t *mtx) { From d6fd11413e1fe33a9bc947d794e880d7d10f7786 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 17:04:24 +0900 Subject: [PATCH 026/352] Define DLLEXPORT when building .jet objects --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 4cb1a65a..65d73db6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -219,7 +219,7 @@ $(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST $(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c $(TESTS_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include ifneq ($(IMPORTLIB),$(SO)) -$(C_OBJS): CPPFLAGS += -DDLLEXPORT +$(C_OBJS) $(C_JET_OBJS): CPPFLAGS += -DDLLEXPORT endif ifndef CC_MM From f41f14366877538b03109ecf346dbff2e21bbb16 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 17:24:08 +0900 Subject: [PATCH 027/352] Replace variable arrays in tests with VARIABLE_ARRAY --- test/unit/hash.c | 4 ++-- test/unit/mallctl.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/unit/hash.c b/test/unit/hash.c index abb394ac..77a8cede 100644 --- a/test/unit/hash.c +++ b/test/unit/hash.c @@ -64,8 +64,8 @@ hash_variant_verify(hash_variant_t variant) { const size_t hashbytes = hash_variant_bits(variant) / 8; uint8_t key[256]; - uint8_t hashes[hashbytes * 256]; - uint8_t final[hashbytes]; + VARIABLE_ARRAY(uint8_t, hashes, hashbytes * 256); + VARIABLE_ARRAY(uint8_t, final, hashbytes); unsigned i; uint32_t computed, expected; diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index cb120497..7a8b55f5 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -285,7 +285,7 @@ TEST_BEGIN(test_arenas_initialized) assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); { - bool initialized[narenas]; + VARIABLE_ARRAY(bool, initialized, narenas); sz = narenas * sizeof(bool); assert_d_eq(mallctl("arenas.initialized", initialized, &sz, From 1ad4a6e9f9ba55c874d0ad63041e09b96b459b1f Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 17:44:42 +0900 Subject: [PATCH 028/352] Add missing $(EXE) to filter TESTS_UNIT_AUX_OBJS --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 65d73db6..839bb08f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -213,7 +213,7 @@ define make-unit-link-dep $(1): TESTS_UNIT_LINK_OBJS += $(2) $(1): $(2) endef -$(foreach test, $(TESTS_UNIT:$(srcroot)test/unit/%.c=$(objroot)test/unit/%$(EXE)), $(eval $(call make-unit-link-dep,$(test),$(filter $(test:%=%_a.$(O)) $(test:%=%_b.$(O)),$(TESTS_UNIT_AUX_OBJS))))) +$(foreach test, $(TESTS_UNIT:$(srcroot)test/unit/%.c=$(objroot)test/unit/%$(EXE)), $(eval $(call make-unit-link-dep,$(test),$(filter $(test:%$(EXE)=%_a.$(O)) $(test:%$(EXE)=%_b.$(O)),$(TESTS_UNIT_AUX_OBJS))))) $(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST $(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST $(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c From 7330c3770af0e5328d749635217387efbbe0ae3c Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 18:00:15 +0900 Subject: [PATCH 029/352] Use C99 varadic macros instead of GCC ones --- test/include/test/test.h | 384 +++++++++++++++++++-------------------- test/unit/util.c | 8 +- 2 files changed, 196 insertions(+), 196 deletions(-) diff --git a/test/include/test/test.h b/test/include/test/test.h index 161fafdf..f55bafce 100644 --- a/test/include/test/test.h +++ b/test/include/test/test.h @@ -1,6 +1,6 @@ #define ASSERT_BUFSIZE 256 -#define assert_cmp(t, a, b, cmp, neg_cmp, pri, fmt...) do { \ +#define assert_cmp(t, a, b, cmp, neg_cmp, pri, ...) do { \ t a_ = (a); \ t b_ = (b); \ if (!(a_ cmp b_)) { \ @@ -12,205 +12,205 @@ "%"pri" "#neg_cmp" %"pri": ", \ __func__, __FILE__, __LINE__, \ #a, #b, a_, b_); \ - malloc_snprintf(message, sizeof(message), fmt); \ + malloc_snprintf(message, sizeof(message), __VA_ARGS__); \ p_test_fail(prefix, message); \ } \ } while (0) -#define assert_ptr_eq(a, b, fmt...) assert_cmp(void *, a, b, ==, \ - !=, "p", fmt) -#define assert_ptr_ne(a, b, fmt...) assert_cmp(void *, a, b, !=, \ - ==, "p", fmt) -#define assert_ptr_null(a, fmt...) assert_cmp(void *, a, NULL, ==, \ - !=, "p", fmt) -#define assert_ptr_not_null(a, fmt...) assert_cmp(void *, a, NULL, !=, \ - ==, "p", fmt) +#define assert_ptr_eq(a, b, ...) assert_cmp(void *, a, b, ==, \ + !=, "p", __VA_ARGS__) +#define assert_ptr_ne(a, b, ...) assert_cmp(void *, a, b, !=, \ + ==, "p", __VA_ARGS__) +#define assert_ptr_null(a, ...) assert_cmp(void *, a, NULL, ==, \ + !=, "p", __VA_ARGS__) +#define assert_ptr_not_null(a, ...) assert_cmp(void *, a, NULL, !=, \ + ==, "p", __VA_ARGS__) -#define assert_c_eq(a, b, fmt...) assert_cmp(char, a, b, ==, !=, "c", fmt) -#define assert_c_ne(a, b, fmt...) assert_cmp(char, a, b, !=, ==, "c", fmt) -#define assert_c_lt(a, b, fmt...) assert_cmp(char, a, b, <, >=, "c", fmt) -#define assert_c_le(a, b, fmt...) assert_cmp(char, a, b, <=, >, "c", fmt) -#define assert_c_ge(a, b, fmt...) assert_cmp(char, a, b, >=, <, "c", fmt) -#define assert_c_gt(a, b, fmt...) assert_cmp(char, a, b, >, <=, "c", fmt) +#define assert_c_eq(a, b, ...) assert_cmp(char, a, b, ==, !=, "c", __VA_ARGS__) +#define assert_c_ne(a, b, ...) assert_cmp(char, a, b, !=, ==, "c", __VA_ARGS__) +#define assert_c_lt(a, b, ...) assert_cmp(char, a, b, <, >=, "c", __VA_ARGS__) +#define assert_c_le(a, b, ...) assert_cmp(char, a, b, <=, >, "c", __VA_ARGS__) +#define assert_c_ge(a, b, ...) assert_cmp(char, a, b, >=, <, "c", __VA_ARGS__) +#define assert_c_gt(a, b, ...) assert_cmp(char, a, b, >, <=, "c", __VA_ARGS__) -#define assert_x_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "#x", fmt) -#define assert_x_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "#x", fmt) -#define assert_x_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "#x", fmt) -#define assert_x_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "#x", fmt) -#define assert_x_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "#x", fmt) -#define assert_x_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "#x", fmt) +#define assert_x_eq(a, b, ...) assert_cmp(int, a, b, ==, !=, "#x", __VA_ARGS__) +#define assert_x_ne(a, b, ...) assert_cmp(int, a, b, !=, ==, "#x", __VA_ARGS__) +#define assert_x_lt(a, b, ...) assert_cmp(int, a, b, <, >=, "#x", __VA_ARGS__) +#define assert_x_le(a, b, ...) assert_cmp(int, a, b, <=, >, "#x", __VA_ARGS__) +#define assert_x_ge(a, b, ...) assert_cmp(int, a, b, >=, <, "#x", __VA_ARGS__) +#define assert_x_gt(a, b, ...) assert_cmp(int, a, b, >, <=, "#x", __VA_ARGS__) -#define assert_d_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "d", fmt) -#define assert_d_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "d", fmt) -#define assert_d_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "d", fmt) -#define assert_d_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "d", fmt) -#define assert_d_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "d", fmt) -#define assert_d_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "d", fmt) +#define assert_d_eq(a, b, ...) assert_cmp(int, a, b, ==, !=, "d", __VA_ARGS__) +#define assert_d_ne(a, b, ...) assert_cmp(int, a, b, !=, ==, "d", __VA_ARGS__) +#define assert_d_lt(a, b, ...) assert_cmp(int, a, b, <, >=, "d", __VA_ARGS__) +#define assert_d_le(a, b, ...) assert_cmp(int, a, b, <=, >, "d", __VA_ARGS__) +#define assert_d_ge(a, b, ...) assert_cmp(int, a, b, >=, <, "d", __VA_ARGS__) +#define assert_d_gt(a, b, ...) assert_cmp(int, a, b, >, <=, "d", __VA_ARGS__) -#define assert_u_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "u", fmt) -#define assert_u_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "u", fmt) -#define assert_u_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "u", fmt) -#define assert_u_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "u", fmt) -#define assert_u_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "u", fmt) -#define assert_u_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "u", fmt) +#define assert_u_eq(a, b, ...) assert_cmp(int, a, b, ==, !=, "u", __VA_ARGS__) +#define assert_u_ne(a, b, ...) assert_cmp(int, a, b, !=, ==, "u", __VA_ARGS__) +#define assert_u_lt(a, b, ...) assert_cmp(int, a, b, <, >=, "u", __VA_ARGS__) +#define assert_u_le(a, b, ...) assert_cmp(int, a, b, <=, >, "u", __VA_ARGS__) +#define assert_u_ge(a, b, ...) assert_cmp(int, a, b, >=, <, "u", __VA_ARGS__) +#define assert_u_gt(a, b, ...) assert_cmp(int, a, b, >, <=, "u", __VA_ARGS__) -#define assert_ld_eq(a, b, fmt...) assert_cmp(long, a, b, ==, \ - !=, "ld", fmt) -#define assert_ld_ne(a, b, fmt...) assert_cmp(long, a, b, !=, \ - ==, "ld", fmt) -#define assert_ld_lt(a, b, fmt...) assert_cmp(long, a, b, <, \ - >=, "ld", fmt) -#define assert_ld_le(a, b, fmt...) assert_cmp(long, a, b, <=, \ - >, "ld", fmt) -#define assert_ld_ge(a, b, fmt...) assert_cmp(long, a, b, >=, \ - <, "ld", fmt) -#define assert_ld_gt(a, b, fmt...) assert_cmp(long, a, b, >, \ - <=, "ld", fmt) +#define assert_ld_eq(a, b, ...) assert_cmp(long, a, b, ==, \ + !=, "ld", __VA_ARGS__) +#define assert_ld_ne(a, b, ...) assert_cmp(long, a, b, !=, \ + ==, "ld", __VA_ARGS__) +#define assert_ld_lt(a, b, ...) assert_cmp(long, a, b, <, \ + >=, "ld", __VA_ARGS__) +#define assert_ld_le(a, b, ...) assert_cmp(long, a, b, <=, \ + >, "ld", __VA_ARGS__) +#define assert_ld_ge(a, b, ...) assert_cmp(long, a, b, >=, \ + <, "ld", __VA_ARGS__) +#define assert_ld_gt(a, b, ...) assert_cmp(long, a, b, >, \ + <=, "ld", __VA_ARGS__) -#define assert_lu_eq(a, b, fmt...) assert_cmp(unsigned long, \ - a, b, ==, !=, "lu", fmt) -#define assert_lu_ne(a, b, fmt...) assert_cmp(unsigned long, \ - a, b, !=, ==, "lu", fmt) -#define assert_lu_lt(a, b, fmt...) assert_cmp(unsigned long, \ - a, b, <, >=, "lu", fmt) -#define assert_lu_le(a, b, fmt...) assert_cmp(unsigned long, \ - a, b, <=, >, "lu", fmt) -#define assert_lu_ge(a, b, fmt...) assert_cmp(unsigned long, \ - a, b, >=, <, "lu", fmt) -#define assert_lu_gt(a, b, fmt...) assert_cmp(unsigned long, \ - a, b, >, <=, "lu", fmt) +#define assert_lu_eq(a, b, ...) assert_cmp(unsigned long, \ + a, b, ==, !=, "lu", __VA_ARGS__) +#define assert_lu_ne(a, b, ...) assert_cmp(unsigned long, \ + a, b, !=, ==, "lu", __VA_ARGS__) +#define assert_lu_lt(a, b, ...) assert_cmp(unsigned long, \ + a, b, <, >=, "lu", __VA_ARGS__) +#define assert_lu_le(a, b, ...) assert_cmp(unsigned long, \ + a, b, <=, >, "lu", __VA_ARGS__) +#define assert_lu_ge(a, b, ...) assert_cmp(unsigned long, \ + a, b, >=, <, "lu", __VA_ARGS__) +#define assert_lu_gt(a, b, ...) assert_cmp(unsigned long, \ + a, b, >, <=, "lu", __VA_ARGS__) -#define assert_qd_eq(a, b, fmt...) assert_cmp(long long, a, b, ==, \ - !=, "qd", fmt) -#define assert_qd_ne(a, b, fmt...) assert_cmp(long long, a, b, !=, \ - ==, "qd", fmt) -#define assert_qd_lt(a, b, fmt...) assert_cmp(long long, a, b, <, \ - >=, "qd", fmt) -#define assert_qd_le(a, b, fmt...) assert_cmp(long long, a, b, <=, \ - >, "qd", fmt) -#define assert_qd_ge(a, b, fmt...) assert_cmp(long long, a, b, >=, \ - <, "qd", fmt) -#define assert_qd_gt(a, b, fmt...) assert_cmp(long long, a, b, >, \ - <=, "qd", fmt) +#define assert_qd_eq(a, b, ...) assert_cmp(long long, a, b, ==, \ + !=, "qd", __VA_ARGS__) +#define assert_qd_ne(a, b, ...) assert_cmp(long long, a, b, !=, \ + ==, "qd", __VA_ARGS__) +#define assert_qd_lt(a, b, ...) assert_cmp(long long, a, b, <, \ + >=, "qd", __VA_ARGS__) +#define assert_qd_le(a, b, ...) assert_cmp(long long, a, b, <=, \ + >, "qd", __VA_ARGS__) +#define assert_qd_ge(a, b, ...) assert_cmp(long long, a, b, >=, \ + <, "qd", __VA_ARGS__) +#define assert_qd_gt(a, b, ...) assert_cmp(long long, a, b, >, \ + <=, "qd", __VA_ARGS__) -#define assert_qu_eq(a, b, fmt...) assert_cmp(unsigned long long, \ - a, b, ==, !=, "qu", fmt) -#define assert_qu_ne(a, b, fmt...) assert_cmp(unsigned long long, \ - a, b, !=, ==, "qu", fmt) -#define assert_qu_lt(a, b, fmt...) assert_cmp(unsigned long long, \ - a, b, <, >=, "qu", fmt) -#define assert_qu_le(a, b, fmt...) assert_cmp(unsigned long long, \ - a, b, <=, >, "qu", fmt) -#define assert_qu_ge(a, b, fmt...) assert_cmp(unsigned long long, \ - a, b, >=, <, "qu", fmt) -#define assert_qu_gt(a, b, fmt...) assert_cmp(unsigned long long, \ - a, b, >, <=, "qu", fmt) +#define assert_qu_eq(a, b, ...) assert_cmp(unsigned long long, \ + a, b, ==, !=, "qu", __VA_ARGS__) +#define assert_qu_ne(a, b, ...) assert_cmp(unsigned long long, \ + a, b, !=, ==, "qu", __VA_ARGS__) +#define assert_qu_lt(a, b, ...) assert_cmp(unsigned long long, \ + a, b, <, >=, "qu", __VA_ARGS__) +#define assert_qu_le(a, b, ...) assert_cmp(unsigned long long, \ + a, b, <=, >, "qu", __VA_ARGS__) +#define assert_qu_ge(a, b, ...) assert_cmp(unsigned long long, \ + a, b, >=, <, "qu", __VA_ARGS__) +#define assert_qu_gt(a, b, ...) assert_cmp(unsigned long long, \ + a, b, >, <=, "qu", __VA_ARGS__) -#define assert_jd_eq(a, b, fmt...) assert_cmp(intmax_t, a, b, ==, \ - !=, "jd", fmt) -#define assert_jd_ne(a, b, fmt...) assert_cmp(intmax_t, a, b, !=, \ - ==, "jd", fmt) -#define assert_jd_lt(a, b, fmt...) assert_cmp(intmax_t, a, b, <, \ - >=, "jd", fmt) -#define assert_jd_le(a, b, fmt...) assert_cmp(intmax_t, a, b, <=, \ - >, "jd", fmt) -#define assert_jd_ge(a, b, fmt...) assert_cmp(intmax_t, a, b, >=, \ - <, "jd", fmt) -#define assert_jd_gt(a, b, fmt...) assert_cmp(intmax_t, a, b, >, \ - <=, "jd", fmt) +#define assert_jd_eq(a, b, ...) assert_cmp(intmax_t, a, b, ==, \ + !=, "jd", __VA_ARGS__) +#define assert_jd_ne(a, b, ...) assert_cmp(intmax_t, a, b, !=, \ + ==, "jd", __VA_ARGS__) +#define assert_jd_lt(a, b, ...) assert_cmp(intmax_t, a, b, <, \ + >=, "jd", __VA_ARGS__) +#define assert_jd_le(a, b, ...) assert_cmp(intmax_t, a, b, <=, \ + >, "jd", __VA_ARGS__) +#define assert_jd_ge(a, b, ...) assert_cmp(intmax_t, a, b, >=, \ + <, "jd", __VA_ARGS__) +#define assert_jd_gt(a, b, ...) assert_cmp(intmax_t, a, b, >, \ + <=, "jd", __VA_ARGS__) -#define assert_ju_eq(a, b, fmt...) assert_cmp(uintmax_t, a, b, ==, \ - !=, "ju", fmt) -#define assert_ju_ne(a, b, fmt...) assert_cmp(uintmax_t, a, b, !=, \ - ==, "ju", fmt) -#define assert_ju_lt(a, b, fmt...) assert_cmp(uintmax_t, a, b, <, \ - >=, "ju", fmt) -#define assert_ju_le(a, b, fmt...) assert_cmp(uintmax_t, a, b, <=, \ - >, "ju", fmt) -#define assert_ju_ge(a, b, fmt...) assert_cmp(uintmax_t, a, b, >=, \ - <, "ju", fmt) -#define assert_ju_gt(a, b, fmt...) assert_cmp(uintmax_t, a, b, >, \ - <=, "ju", fmt) +#define assert_ju_eq(a, b, ...) assert_cmp(uintmax_t, a, b, ==, \ + !=, "ju", __VA_ARGS__) +#define assert_ju_ne(a, b, ...) assert_cmp(uintmax_t, a, b, !=, \ + ==, "ju", __VA_ARGS__) +#define assert_ju_lt(a, b, ...) assert_cmp(uintmax_t, a, b, <, \ + >=, "ju", __VA_ARGS__) +#define assert_ju_le(a, b, ...) assert_cmp(uintmax_t, a, b, <=, \ + >, "ju", __VA_ARGS__) +#define assert_ju_ge(a, b, ...) assert_cmp(uintmax_t, a, b, >=, \ + <, "ju", __VA_ARGS__) +#define assert_ju_gt(a, b, ...) assert_cmp(uintmax_t, a, b, >, \ + <=, "ju", __VA_ARGS__) -#define assert_zd_eq(a, b, fmt...) assert_cmp(ssize_t, a, b, ==, \ - !=, "zd", fmt) -#define assert_zd_ne(a, b, fmt...) assert_cmp(ssize_t, a, b, !=, \ - ==, "zd", fmt) -#define assert_zd_lt(a, b, fmt...) assert_cmp(ssize_t, a, b, <, \ - >=, "zd", fmt) -#define assert_zd_le(a, b, fmt...) assert_cmp(ssize_t, a, b, <=, \ - >, "zd", fmt) -#define assert_zd_ge(a, b, fmt...) assert_cmp(ssize_t, a, b, >=, \ - <, "zd", fmt) -#define assert_zd_gt(a, b, fmt...) assert_cmp(ssize_t, a, b, >, \ - <=, "zd", fmt) +#define assert_zd_eq(a, b, ...) assert_cmp(ssize_t, a, b, ==, \ + !=, "zd", __VA_ARGS__) +#define assert_zd_ne(a, b, ...) assert_cmp(ssize_t, a, b, !=, \ + ==, "zd", __VA_ARGS__) +#define assert_zd_lt(a, b, ...) assert_cmp(ssize_t, a, b, <, \ + >=, "zd", __VA_ARGS__) +#define assert_zd_le(a, b, ...) assert_cmp(ssize_t, a, b, <=, \ + >, "zd", __VA_ARGS__) +#define assert_zd_ge(a, b, ...) assert_cmp(ssize_t, a, b, >=, \ + <, "zd", __VA_ARGS__) +#define assert_zd_gt(a, b, ...) assert_cmp(ssize_t, a, b, >, \ + <=, "zd", __VA_ARGS__) -#define assert_zu_eq(a, b, fmt...) assert_cmp(size_t, a, b, ==, \ - !=, "zu", fmt) -#define assert_zu_ne(a, b, fmt...) assert_cmp(size_t, a, b, !=, \ - ==, "zu", fmt) -#define assert_zu_lt(a, b, fmt...) assert_cmp(size_t, a, b, <, \ - >=, "zu", fmt) -#define assert_zu_le(a, b, fmt...) assert_cmp(size_t, a, b, <=, \ - >, "zu", fmt) -#define assert_zu_ge(a, b, fmt...) assert_cmp(size_t, a, b, >=, \ - <, "zu", fmt) -#define assert_zu_gt(a, b, fmt...) assert_cmp(size_t, a, b, >, \ - <=, "zu", fmt) +#define assert_zu_eq(a, b, ...) assert_cmp(size_t, a, b, ==, \ + !=, "zu", __VA_ARGS__) +#define assert_zu_ne(a, b, ...) assert_cmp(size_t, a, b, !=, \ + ==, "zu", __VA_ARGS__) +#define assert_zu_lt(a, b, ...) assert_cmp(size_t, a, b, <, \ + >=, "zu", __VA_ARGS__) +#define assert_zu_le(a, b, ...) assert_cmp(size_t, a, b, <=, \ + >, "zu", __VA_ARGS__) +#define assert_zu_ge(a, b, ...) assert_cmp(size_t, a, b, >=, \ + <, "zu", __VA_ARGS__) +#define assert_zu_gt(a, b, ...) assert_cmp(size_t, a, b, >, \ + <=, "zu", __VA_ARGS__) -#define assert_d32_eq(a, b, fmt...) assert_cmp(int32_t, a, b, ==, \ - !=, PRId32, fmt) -#define assert_d32_ne(a, b, fmt...) assert_cmp(int32_t, a, b, !=, \ - ==, PRId32, fmt) -#define assert_d32_lt(a, b, fmt...) assert_cmp(int32_t, a, b, <, \ - >=, PRId32, fmt) -#define assert_d32_le(a, b, fmt...) assert_cmp(int32_t, a, b, <=, \ - >, PRId32, fmt) -#define assert_d32_ge(a, b, fmt...) assert_cmp(int32_t, a, b, >=, \ - <, PRId32, fmt) -#define assert_d32_gt(a, b, fmt...) assert_cmp(int32_t, a, b, >, \ - <=, PRId32, fmt) +#define assert_d32_eq(a, b, ...) assert_cmp(int32_t, a, b, ==, \ + !=, PRId32, __VA_ARGS__) +#define assert_d32_ne(a, b, ...) assert_cmp(int32_t, a, b, !=, \ + ==, PRId32, __VA_ARGS__) +#define assert_d32_lt(a, b, ...) assert_cmp(int32_t, a, b, <, \ + >=, PRId32, __VA_ARGS__) +#define assert_d32_le(a, b, ...) assert_cmp(int32_t, a, b, <=, \ + >, PRId32, __VA_ARGS__) +#define assert_d32_ge(a, b, ...) assert_cmp(int32_t, a, b, >=, \ + <, PRId32, __VA_ARGS__) +#define assert_d32_gt(a, b, ...) assert_cmp(int32_t, a, b, >, \ + <=, PRId32, __VA_ARGS__) -#define assert_u32_eq(a, b, fmt...) assert_cmp(uint32_t, a, b, ==, \ - !=, PRIu32, fmt) -#define assert_u32_ne(a, b, fmt...) assert_cmp(uint32_t, a, b, !=, \ - ==, PRIu32, fmt) -#define assert_u32_lt(a, b, fmt...) assert_cmp(uint32_t, a, b, <, \ - >=, PRIu32, fmt) -#define assert_u32_le(a, b, fmt...) assert_cmp(uint32_t, a, b, <=, \ - >, PRIu32, fmt) -#define assert_u32_ge(a, b, fmt...) assert_cmp(uint32_t, a, b, >=, \ - <, PRIu32, fmt) -#define assert_u32_gt(a, b, fmt...) assert_cmp(uint32_t, a, b, >, \ - <=, PRIu32, fmt) +#define assert_u32_eq(a, b, ...) assert_cmp(uint32_t, a, b, ==, \ + !=, PRIu32, __VA_ARGS__) +#define assert_u32_ne(a, b, ...) assert_cmp(uint32_t, a, b, !=, \ + ==, PRIu32, __VA_ARGS__) +#define assert_u32_lt(a, b, ...) assert_cmp(uint32_t, a, b, <, \ + >=, PRIu32, __VA_ARGS__) +#define assert_u32_le(a, b, ...) assert_cmp(uint32_t, a, b, <=, \ + >, PRIu32, __VA_ARGS__) +#define assert_u32_ge(a, b, ...) assert_cmp(uint32_t, a, b, >=, \ + <, PRIu32, __VA_ARGS__) +#define assert_u32_gt(a, b, ...) assert_cmp(uint32_t, a, b, >, \ + <=, PRIu32, __VA_ARGS__) -#define assert_d64_eq(a, b, fmt...) assert_cmp(int64_t, a, b, ==, \ - !=, PRId64, fmt) -#define assert_d64_ne(a, b, fmt...) assert_cmp(int64_t, a, b, !=, \ - ==, PRId64, fmt) -#define assert_d64_lt(a, b, fmt...) assert_cmp(int64_t, a, b, <, \ - >=, PRId64, fmt) -#define assert_d64_le(a, b, fmt...) assert_cmp(int64_t, a, b, <=, \ - >, PRId64, fmt) -#define assert_d64_ge(a, b, fmt...) assert_cmp(int64_t, a, b, >=, \ - <, PRId64, fmt) -#define assert_d64_gt(a, b, fmt...) assert_cmp(int64_t, a, b, >, \ - <=, PRId64, fmt) +#define assert_d64_eq(a, b, ...) assert_cmp(int64_t, a, b, ==, \ + !=, PRId64, __VA_ARGS__) +#define assert_d64_ne(a, b, ...) assert_cmp(int64_t, a, b, !=, \ + ==, PRId64, __VA_ARGS__) +#define assert_d64_lt(a, b, ...) assert_cmp(int64_t, a, b, <, \ + >=, PRId64, __VA_ARGS__) +#define assert_d64_le(a, b, ...) assert_cmp(int64_t, a, b, <=, \ + >, PRId64, __VA_ARGS__) +#define assert_d64_ge(a, b, ...) assert_cmp(int64_t, a, b, >=, \ + <, PRId64, __VA_ARGS__) +#define assert_d64_gt(a, b, ...) assert_cmp(int64_t, a, b, >, \ + <=, PRId64, __VA_ARGS__) -#define assert_u64_eq(a, b, fmt...) assert_cmp(uint64_t, a, b, ==, \ - !=, PRIu64, fmt) -#define assert_u64_ne(a, b, fmt...) assert_cmp(uint64_t, a, b, !=, \ - ==, PRIu64, fmt) -#define assert_u64_lt(a, b, fmt...) assert_cmp(uint64_t, a, b, <, \ - >=, PRIu64, fmt) -#define assert_u64_le(a, b, fmt...) assert_cmp(uint64_t, a, b, <=, \ - >, PRIu64, fmt) -#define assert_u64_ge(a, b, fmt...) assert_cmp(uint64_t, a, b, >=, \ - <, PRIu64, fmt) -#define assert_u64_gt(a, b, fmt...) assert_cmp(uint64_t, a, b, >, \ - <=, PRIu64, fmt) +#define assert_u64_eq(a, b, ...) assert_cmp(uint64_t, a, b, ==, \ + !=, PRIu64, __VA_ARGS__) +#define assert_u64_ne(a, b, ...) assert_cmp(uint64_t, a, b, !=, \ + ==, PRIu64, __VA_ARGS__) +#define assert_u64_lt(a, b, ...) assert_cmp(uint64_t, a, b, <, \ + >=, PRIu64, __VA_ARGS__) +#define assert_u64_le(a, b, ...) assert_cmp(uint64_t, a, b, <=, \ + >, PRIu64, __VA_ARGS__) +#define assert_u64_ge(a, b, ...) assert_cmp(uint64_t, a, b, >=, \ + <, PRIu64, __VA_ARGS__) +#define assert_u64_gt(a, b, ...) assert_cmp(uint64_t, a, b, >, \ + <=, PRIu64, __VA_ARGS__) -#define assert_b_eq(a, b, fmt...) do { \ +#define assert_b_eq(a, b, ...) do { \ bool a_ = (a); \ bool b_ = (b); \ if (!(a_ == b_)) { \ @@ -222,11 +222,11 @@ __func__, __FILE__, __LINE__, \ #a, #b, a_ ? "true" : "false", \ b_ ? "true" : "false"); \ - malloc_snprintf(message, sizeof(message), fmt); \ + malloc_snprintf(message, sizeof(message), __VA_ARGS__); \ p_test_fail(prefix, message); \ } \ } while (0) -#define assert_b_ne(a, b, fmt...) do { \ +#define assert_b_ne(a, b, ...) do { \ bool a_ = (a); \ bool b_ = (b); \ if (!(a_ != b_)) { \ @@ -238,14 +238,14 @@ __func__, __FILE__, __LINE__, \ #a, #b, a_ ? "true" : "false", \ b_ ? "true" : "false"); \ - malloc_snprintf(message, sizeof(message), fmt); \ + malloc_snprintf(message, sizeof(message), __VA_ARGS__); \ p_test_fail(prefix, message); \ } \ } while (0) -#define assert_true(a, fmt...) assert_b_eq(a, true, fmt) -#define assert_false(a, fmt...) assert_b_eq(a, false, fmt) +#define assert_true(a, ...) assert_b_eq(a, true, __VA_ARGS__) +#define assert_false(a, ...) assert_b_eq(a, false, __VA_ARGS__) -#define assert_str_eq(a, b, fmt...) do { \ +#define assert_str_eq(a, b, ...) do { \ if (strcmp((a), (b))) { \ char prefix[ASSERT_BUFSIZE]; \ char message[ASSERT_BUFSIZE]; \ @@ -254,11 +254,11 @@ "(%s) same as (%s) --> " \ "\"%s\" differs from \"%s\": ", \ __func__, __FILE__, __LINE__, #a, #b, a, b); \ - malloc_snprintf(message, sizeof(message), fmt); \ + malloc_snprintf(message, sizeof(message), __VA_ARGS__); \ p_test_fail(prefix, message); \ } \ } while (0) -#define assert_str_ne(a, b, fmt...) do { \ +#define assert_str_ne(a, b, ...) do { \ if (!strcmp((a), (b))) { \ char prefix[ASSERT_BUFSIZE]; \ char message[ASSERT_BUFSIZE]; \ @@ -267,18 +267,18 @@ "(%s) differs from (%s) --> " \ "\"%s\" same as \"%s\": ", \ __func__, __FILE__, __LINE__, #a, #b, a, b); \ - malloc_snprintf(message, sizeof(message), fmt); \ + malloc_snprintf(message, sizeof(message), __VA_ARGS__); \ p_test_fail(prefix, message); \ } \ } while (0) -#define assert_not_reached(fmt...) do { \ +#define assert_not_reached(...) do { \ char prefix[ASSERT_BUFSIZE]; \ char message[ASSERT_BUFSIZE]; \ malloc_snprintf(prefix, sizeof(prefix), \ "%s:%s:%d: Unreachable code reached: ", \ __func__, __FILE__, __LINE__); \ - malloc_snprintf(message, sizeof(message), fmt); \ + malloc_snprintf(message, sizeof(message), __VA_ARGS__); \ p_test_fail(prefix, message); \ } while (0) @@ -308,8 +308,8 @@ label_test_end: \ p_test_fini(); \ } -#define test(tests...) \ - p_test(tests, NULL) +#define test(...) \ + p_test(__VA_ARGS__, NULL) #define test_skip_if(e) do { \ if (e) { \ diff --git a/test/unit/util.c b/test/unit/util.c index dc3cfe8a..c11d5984 100644 --- a/test/unit/util.c +++ b/test/unit/util.c @@ -141,8 +141,8 @@ TEST_BEGIN(test_malloc_snprintf_truncated) char buf[BUFLEN]; int result; size_t len; -#define TEST(expected_str_untruncated, fmt...) do { \ - result = malloc_snprintf(buf, len, fmt); \ +#define TEST(expected_str_untruncated, ...) do { \ + result = malloc_snprintf(buf, len, __VA_ARGS__); \ assert_d_eq(strncmp(buf, expected_str_untruncated, len-1), 0, \ "Unexpected string inequality (\"%s\" vs \"%s\")", \ buf, expected_str_untruncated); \ @@ -173,8 +173,8 @@ TEST_BEGIN(test_malloc_snprintf) #define BUFLEN 128 char buf[BUFLEN]; int result; -#define TEST(expected_str, fmt...) do { \ - result = malloc_snprintf(buf, sizeof(buf), fmt); \ +#define TEST(expected_str, ...) do { \ + result = malloc_snprintf(buf, sizeof(buf), __VA_ARGS__); \ assert_str_eq(buf, expected_str, "Unexpected output"); \ assert_d_eq(result, strlen(expected_str), "Unexpected result"); \ } while (0) From 86e2e703ffb3cc17e05af816df8895db62a9272e Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 18:01:21 +0900 Subject: [PATCH 030/352] Rename "small" local variable, because windows headers #define it --- test/unit/stats.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/unit/stats.c b/test/unit/stats.c index ab87b29b..78c78cd5 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -97,7 +97,7 @@ TEST_END TEST_BEGIN(test_stats_arenas_summary) { unsigned arena; - void *small, *large; + void *little, *large; uint64_t epoch; size_t sz; int expected = config_stats ? 0 : ENOENT; @@ -108,8 +108,8 @@ TEST_BEGIN(test_stats_arenas_summary) assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), 0, "Unexpected mallctl() failure"); - small = mallocx(SMALL_MAXCLASS, 0); - assert_ptr_not_null(small, "Unexpected mallocx() failure"); + little = mallocx(SMALL_MAXCLASS, 0); + assert_ptr_not_null(little, "Unexpected mallocx() failure"); large = mallocx(arena_maxclass, 0); assert_ptr_not_null(large, "Unexpected mallocx() failure"); @@ -137,7 +137,7 @@ TEST_BEGIN(test_stats_arenas_summary) "nmadvise should be no greater than purged"); } - dallocx(small, 0); + dallocx(little, 0); dallocx(large, 0); } TEST_END From 3a730dfd5062ecd6fc46b68f28342e14b461f560 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 18:13:21 +0900 Subject: [PATCH 031/352] Avoid pointer arithmetic on void* in test/integration/rallocx.c --- test/integration/rallocx.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/integration/rallocx.c b/test/integration/rallocx.c index ee21aedf..e78e02f3 100644 --- a/test/integration/rallocx.c +++ b/test/integration/rallocx.c @@ -95,7 +95,8 @@ TEST_BEGIN(test_zero) "Expected zeroed memory"); } if (psz != qsz) { - memset(q+psz, FILL_BYTE, qsz-psz); + memset((void *)(uintptr_t)q+psz, FILL_BYTE, + qsz-psz); psz = qsz; } p = q; @@ -159,8 +160,9 @@ TEST_BEGIN(test_lg_align_and_zero) } else { assert_false(validate_fill(q, 0, 0, MAX_VALIDATE), "Expected zeroed memory"); - assert_false(validate_fill(q+sz-MAX_VALIDATE, 0, 0, - MAX_VALIDATE), "Expected zeroed memory"); + assert_false(validate_fill( + (void *)(uintptr_t)q+sz-MAX_VALIDATE, + 0, 0, MAX_VALIDATE), "Expected zeroed memory"); } p = q; } From a9df1ae622d0eb91a26208c03c51d0c518cce146 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 16:34:02 +0900 Subject: [PATCH 032/352] Use ULL prefix instead of LLU for unsigned long longs MSVC only supports the former. --- include/jemalloc/internal/hash.h | 8 +- test/src/SFMT.c | 2 +- test/unit/SFMT.c | 2000 +++++++++++++++--------------- 3 files changed, 1005 insertions(+), 1005 deletions(-) diff --git a/include/jemalloc/internal/hash.h b/include/jemalloc/internal/hash.h index c7183ede..f2b3a16c 100644 --- a/include/jemalloc/internal/hash.h +++ b/include/jemalloc/internal/hash.h @@ -76,9 +76,9 @@ hash_fmix_64(uint64_t k) { k ^= k >> 33; - k *= QU(0xff51afd7ed558ccdLLU); + k *= QU(0xff51afd7ed558ccdULL); k ^= k >> 33; - k *= QU(0xc4ceb9fe1a85ec53LLU); + k *= QU(0xc4ceb9fe1a85ec53ULL); k ^= k >> 33; return (k); @@ -247,8 +247,8 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t h1 = seed; uint64_t h2 = seed; - const uint64_t c1 = QU(0x87c37b91114253d5LLU); - const uint64_t c2 = QU(0x4cf5ad432745937fLLU); + const uint64_t c1 = QU(0x87c37b91114253d5ULL); + const uint64_t c2 = QU(0x4cf5ad432745937fULL); /* body */ { diff --git a/test/src/SFMT.c b/test/src/SFMT.c index e6f8deec..d2cc9d1c 100644 --- a/test/src/SFMT.c +++ b/test/src/SFMT.c @@ -511,7 +511,7 @@ uint64_t gen_rand64(sfmt_t *ctx) { uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit) { uint64_t ret, above; - above = 0xffffffffffffffffLLU - (0xffffffffffffffffLLU % limit); + above = 0xffffffffffffffffULL - (0xffffffffffffffffULL % limit); while (1) { ret = gen_rand64(ctx); if (ret < above) { diff --git a/test/unit/SFMT.c b/test/unit/SFMT.c index c57bd68d..0ad9c233 100644 --- a/test/unit/SFMT.c +++ b/test/unit/SFMT.c @@ -445,1008 +445,1008 @@ static const uint32_t init_by_array_32_expected[] = { 2750138839U, 3518055702U, 733072558U, 4169325400U, 788493625U }; static const uint64_t init_gen_rand_64_expected[] = { - QU(16924766246869039260LLU), QU( 8201438687333352714LLU), - QU( 2265290287015001750LLU), QU(18397264611805473832LLU), - QU( 3375255223302384358LLU), QU( 6345559975416828796LLU), - QU(18229739242790328073LLU), QU( 7596792742098800905LLU), - QU( 255338647169685981LLU), QU( 2052747240048610300LLU), - QU(18328151576097299343LLU), QU(12472905421133796567LLU), - QU(11315245349717600863LLU), QU(16594110197775871209LLU), - QU(15708751964632456450LLU), QU(10452031272054632535LLU), - QU(11097646720811454386LLU), QU( 4556090668445745441LLU), - QU(17116187693090663106LLU), QU(14931526836144510645LLU), - QU( 9190752218020552591LLU), QU( 9625800285771901401LLU), - QU(13995141077659972832LLU), QU( 5194209094927829625LLU), - QU( 4156788379151063303LLU), QU( 8523452593770139494LLU), - QU(14082382103049296727LLU), QU( 2462601863986088483LLU), - QU( 3030583461592840678LLU), QU( 5221622077872827681LLU), - QU( 3084210671228981236LLU), QU(13956758381389953823LLU), - QU(13503889856213423831LLU), QU(15696904024189836170LLU), - QU( 4612584152877036206LLU), QU( 6231135538447867881LLU), - QU(10172457294158869468LLU), QU( 6452258628466708150LLU), - QU(14044432824917330221LLU), QU( 370168364480044279LLU), - QU(10102144686427193359LLU), QU( 667870489994776076LLU), - QU( 2732271956925885858LLU), QU(18027788905977284151LLU), - QU(15009842788582923859LLU), QU( 7136357960180199542LLU), - QU(15901736243475578127LLU), QU(16951293785352615701LLU), - QU(10551492125243691632LLU), QU(17668869969146434804LLU), - QU(13646002971174390445LLU), QU( 9804471050759613248LLU), - QU( 5511670439655935493LLU), QU(18103342091070400926LLU), - QU(17224512747665137533LLU), QU(15534627482992618168LLU), - QU( 1423813266186582647LLU), QU(15821176807932930024LLU), - QU( 30323369733607156LLU), QU(11599382494723479403LLU), - QU( 653856076586810062LLU), QU( 3176437395144899659LLU), - QU(14028076268147963917LLU), QU(16156398271809666195LLU), - QU( 3166955484848201676LLU), QU( 5746805620136919390LLU), - QU(17297845208891256593LLU), QU(11691653183226428483LLU), - QU(17900026146506981577LLU), QU(15387382115755971042LLU), - QU(16923567681040845943LLU), QU( 8039057517199388606LLU), - QU(11748409241468629263LLU), QU( 794358245539076095LLU), - QU(13438501964693401242LLU), QU(14036803236515618962LLU), - QU( 5252311215205424721LLU), QU(17806589612915509081LLU), - QU( 6802767092397596006LLU), QU(14212120431184557140LLU), - QU( 1072951366761385712LLU), QU(13098491780722836296LLU), - QU( 9466676828710797353LLU), QU(12673056849042830081LLU), - QU(12763726623645357580LLU), QU(16468961652999309493LLU), - QU(15305979875636438926LLU), QU(17444713151223449734LLU), - QU( 5692214267627883674LLU), QU(13049589139196151505LLU), - QU( 880115207831670745LLU), QU( 1776529075789695498LLU), - QU(16695225897801466485LLU), QU(10666901778795346845LLU), - QU( 6164389346722833869LLU), QU( 2863817793264300475LLU), - QU( 9464049921886304754LLU), QU( 3993566636740015468LLU), - QU( 9983749692528514136LLU), QU(16375286075057755211LLU), - QU(16042643417005440820LLU), QU(11445419662923489877LLU), - QU( 7999038846885158836LLU), QU( 6721913661721511535LLU), - QU( 5363052654139357320LLU), QU( 1817788761173584205LLU), - QU(13290974386445856444LLU), QU( 4650350818937984680LLU), - QU( 8219183528102484836LLU), QU( 1569862923500819899LLU), - QU( 4189359732136641860LLU), QU(14202822961683148583LLU), - QU( 4457498315309429058LLU), QU(13089067387019074834LLU), - QU(11075517153328927293LLU), QU(10277016248336668389LLU), - QU( 7070509725324401122LLU), QU(17808892017780289380LLU), - QU(13143367339909287349LLU), QU( 1377743745360085151LLU), - QU( 5749341807421286485LLU), QU(14832814616770931325LLU), - QU( 7688820635324359492LLU), QU(10960474011539770045LLU), - QU( 81970066653179790LLU), QU(12619476072607878022LLU), - QU( 4419566616271201744LLU), QU(15147917311750568503LLU), - QU( 5549739182852706345LLU), QU( 7308198397975204770LLU), - QU(13580425496671289278LLU), QU(17070764785210130301LLU), - QU( 8202832846285604405LLU), QU( 6873046287640887249LLU), - QU( 6927424434308206114LLU), QU( 6139014645937224874LLU), - QU(10290373645978487639LLU), QU(15904261291701523804LLU), - QU( 9628743442057826883LLU), QU(18383429096255546714LLU), - QU( 4977413265753686967LLU), QU( 7714317492425012869LLU), - QU( 9025232586309926193LLU), QU(14627338359776709107LLU), - QU(14759849896467790763LLU), QU(10931129435864423252LLU), - QU( 4588456988775014359LLU), QU(10699388531797056724LLU), - QU( 468652268869238792LLU), QU( 5755943035328078086LLU), - QU( 2102437379988580216LLU), QU( 9986312786506674028LLU), - QU( 2654207180040945604LLU), QU( 8726634790559960062LLU), - QU( 100497234871808137LLU), QU( 2800137176951425819LLU), - QU( 6076627612918553487LLU), QU( 5780186919186152796LLU), - QU( 8179183595769929098LLU), QU( 6009426283716221169LLU), - QU( 2796662551397449358LLU), QU( 1756961367041986764LLU), - QU( 6972897917355606205LLU), QU(14524774345368968243LLU), - QU( 2773529684745706940LLU), QU( 4853632376213075959LLU), - QU( 4198177923731358102LLU), QU( 8271224913084139776LLU), - QU( 2741753121611092226LLU), QU(16782366145996731181LLU), - QU(15426125238972640790LLU), QU(13595497100671260342LLU), - QU( 3173531022836259898LLU), QU( 6573264560319511662LLU), - QU(18041111951511157441LLU), QU( 2351433581833135952LLU), - QU( 3113255578908173487LLU), QU( 1739371330877858784LLU), - QU(16046126562789165480LLU), QU( 8072101652214192925LLU), - QU(15267091584090664910LLU), QU( 9309579200403648940LLU), - QU( 5218892439752408722LLU), QU(14492477246004337115LLU), - QU(17431037586679770619LLU), QU( 7385248135963250480LLU), - QU( 9580144956565560660LLU), QU( 4919546228040008720LLU), - QU(15261542469145035584LLU), QU(18233297270822253102LLU), - QU( 5453248417992302857LLU), QU( 9309519155931460285LLU), - QU(10342813012345291756LLU), QU(15676085186784762381LLU), - QU(15912092950691300645LLU), QU( 9371053121499003195LLU), - QU( 9897186478226866746LLU), QU(14061858287188196327LLU), - QU( 122575971620788119LLU), QU(12146750969116317754LLU), - QU( 4438317272813245201LLU), QU( 8332576791009527119LLU), - QU(13907785691786542057LLU), QU(10374194887283287467LLU), - QU( 2098798755649059566LLU), QU( 3416235197748288894LLU), - QU( 8688269957320773484LLU), QU( 7503964602397371571LLU), - QU(16724977015147478236LLU), QU( 9461512855439858184LLU), - QU(13259049744534534727LLU), QU( 3583094952542899294LLU), - QU( 8764245731305528292LLU), QU(13240823595462088985LLU), - QU(13716141617617910448LLU), QU(18114969519935960955LLU), - QU( 2297553615798302206LLU), QU( 4585521442944663362LLU), - QU(17776858680630198686LLU), QU( 4685873229192163363LLU), - QU( 152558080671135627LLU), QU(15424900540842670088LLU), - QU(13229630297130024108LLU), QU(17530268788245718717LLU), - QU(16675633913065714144LLU), QU( 3158912717897568068LLU), - QU(15399132185380087288LLU), QU( 7401418744515677872LLU), - QU(13135412922344398535LLU), QU( 6385314346100509511LLU), - QU(13962867001134161139LLU), QU(10272780155442671999LLU), - QU(12894856086597769142LLU), QU(13340877795287554994LLU), - QU(12913630602094607396LLU), QU(12543167911119793857LLU), - QU(17343570372251873096LLU), QU(10959487764494150545LLU), - QU( 6966737953093821128LLU), QU(13780699135496988601LLU), - QU( 4405070719380142046LLU), QU(14923788365607284982LLU), - QU( 2869487678905148380LLU), QU( 6416272754197188403LLU), - QU(15017380475943612591LLU), QU( 1995636220918429487LLU), - QU( 3402016804620122716LLU), QU(15800188663407057080LLU), - QU(11362369990390932882LLU), QU(15262183501637986147LLU), - QU(10239175385387371494LLU), QU( 9352042420365748334LLU), - QU( 1682457034285119875LLU), QU( 1724710651376289644LLU), - QU( 2038157098893817966LLU), QU( 9897825558324608773LLU), - QU( 1477666236519164736LLU), QU(16835397314511233640LLU), - QU(10370866327005346508LLU), QU(10157504370660621982LLU), - QU(12113904045335882069LLU), QU(13326444439742783008LLU), - QU(11302769043000765804LLU), QU(13594979923955228484LLU), - QU(11779351762613475968LLU), QU( 3786101619539298383LLU), - QU( 8021122969180846063LLU), QU(15745904401162500495LLU), - QU(10762168465993897267LLU), QU(13552058957896319026LLU), - QU(11200228655252462013LLU), QU( 5035370357337441226LLU), - QU( 7593918984545500013LLU), QU( 5418554918361528700LLU), - QU( 4858270799405446371LLU), QU( 9974659566876282544LLU), - QU(18227595922273957859LLU), QU( 2772778443635656220LLU), - QU(14285143053182085385LLU), QU( 9939700992429600469LLU), - QU(12756185904545598068LLU), QU( 2020783375367345262LLU), - QU( 57026775058331227LLU), QU( 950827867930065454LLU), - QU( 6602279670145371217LLU), QU( 2291171535443566929LLU), - QU( 5832380724425010313LLU), QU( 1220343904715982285LLU), - QU(17045542598598037633LLU), QU(15460481779702820971LLU), - QU(13948388779949365130LLU), QU(13975040175430829518LLU), - QU(17477538238425541763LLU), QU(11104663041851745725LLU), - QU(15860992957141157587LLU), QU(14529434633012950138LLU), - QU( 2504838019075394203LLU), QU( 7512113882611121886LLU), - QU( 4859973559980886617LLU), QU( 1258601555703250219LLU), - QU(15594548157514316394LLU), QU( 4516730171963773048LLU), - QU(11380103193905031983LLU), QU( 6809282239982353344LLU), - QU(18045256930420065002LLU), QU( 2453702683108791859LLU), - QU( 977214582986981460LLU), QU( 2006410402232713466LLU), - QU( 6192236267216378358LLU), QU( 3429468402195675253LLU), - QU(18146933153017348921LLU), QU(17369978576367231139LLU), - QU( 1246940717230386603LLU), QU(11335758870083327110LLU), - QU(14166488801730353682LLU), QU( 9008573127269635732LLU), - QU(10776025389820643815LLU), QU(15087605441903942962LLU), - QU( 1359542462712147922LLU), QU(13898874411226454206LLU), - QU(17911176066536804411LLU), QU( 9435590428600085274LLU), - QU( 294488509967864007LLU), QU( 8890111397567922046LLU), - QU( 7987823476034328778LLU), QU(13263827582440967651LLU), - QU( 7503774813106751573LLU), QU(14974747296185646837LLU), - QU( 8504765037032103375LLU), QU(17340303357444536213LLU), - QU( 7704610912964485743LLU), QU( 8107533670327205061LLU), - QU( 9062969835083315985LLU), QU(16968963142126734184LLU), - QU(12958041214190810180LLU), QU( 2720170147759570200LLU), - QU( 2986358963942189566LLU), QU(14884226322219356580LLU), - QU( 286224325144368520LLU), QU(11313800433154279797LLU), - QU(18366849528439673248LLU), QU(17899725929482368789LLU), - QU( 3730004284609106799LLU), QU( 1654474302052767205LLU), - QU( 5006698007047077032LLU), QU( 8196893913601182838LLU), - QU(15214541774425211640LLU), QU(17391346045606626073LLU), - QU( 8369003584076969089LLU), QU( 3939046733368550293LLU), - QU(10178639720308707785LLU), QU( 2180248669304388697LLU), - QU( 62894391300126322LLU), QU( 9205708961736223191LLU), - QU( 6837431058165360438LLU), QU( 3150743890848308214LLU), - QU(17849330658111464583LLU), QU(12214815643135450865LLU), - QU(13410713840519603402LLU), QU( 3200778126692046802LLU), - QU(13354780043041779313LLU), QU( 800850022756886036LLU), - QU(15660052933953067433LLU), QU( 6572823544154375676LLU), - QU(11030281857015819266LLU), QU(12682241941471433835LLU), - QU(11654136407300274693LLU), QU( 4517795492388641109LLU), - QU( 9757017371504524244LLU), QU(17833043400781889277LLU), - QU(12685085201747792227LLU), QU(10408057728835019573LLU), - QU( 98370418513455221LLU), QU( 6732663555696848598LLU), - QU(13248530959948529780LLU), QU( 3530441401230622826LLU), - QU(18188251992895660615LLU), QU( 1847918354186383756LLU), - QU( 1127392190402660921LLU), QU(11293734643143819463LLU), - QU( 3015506344578682982LLU), QU(13852645444071153329LLU), - QU( 2121359659091349142LLU), QU( 1294604376116677694LLU), - QU( 5616576231286352318LLU), QU( 7112502442954235625LLU), - QU(11676228199551561689LLU), QU(12925182803007305359LLU), - QU( 7852375518160493082LLU), QU( 1136513130539296154LLU), - QU( 5636923900916593195LLU), QU( 3221077517612607747LLU), - QU(17784790465798152513LLU), QU( 3554210049056995938LLU), - QU(17476839685878225874LLU), QU( 3206836372585575732LLU), - QU( 2765333945644823430LLU), QU(10080070903718799528LLU), - QU( 5412370818878286353LLU), QU( 9689685887726257728LLU), - QU( 8236117509123533998LLU), QU( 1951139137165040214LLU), - QU( 4492205209227980349LLU), QU(16541291230861602967LLU), - QU( 1424371548301437940LLU), QU( 9117562079669206794LLU), - QU(14374681563251691625LLU), QU(13873164030199921303LLU), - QU( 6680317946770936731LLU), QU(15586334026918276214LLU), - QU(10896213950976109802LLU), QU( 9506261949596413689LLU), - QU( 9903949574308040616LLU), QU( 6038397344557204470LLU), - QU( 174601465422373648LLU), QU(15946141191338238030LLU), - QU(17142225620992044937LLU), QU( 7552030283784477064LLU), - QU( 2947372384532947997LLU), QU( 510797021688197711LLU), - QU( 4962499439249363461LLU), QU( 23770320158385357LLU), - QU( 959774499105138124LLU), QU( 1468396011518788276LLU), - QU( 2015698006852312308LLU), QU( 4149400718489980136LLU), - QU( 5992916099522371188LLU), QU(10819182935265531076LLU), - QU(16189787999192351131LLU), QU( 342833961790261950LLU), - QU(12470830319550495336LLU), QU(18128495041912812501LLU), - QU( 1193600899723524337LLU), QU( 9056793666590079770LLU), - QU( 2154021227041669041LLU), QU( 4963570213951235735LLU), - QU( 4865075960209211409LLU), QU( 2097724599039942963LLU), - QU( 2024080278583179845LLU), QU(11527054549196576736LLU), - QU(10650256084182390252LLU), QU( 4808408648695766755LLU), - QU( 1642839215013788844LLU), QU(10607187948250398390LLU), - QU( 7076868166085913508LLU), QU( 730522571106887032LLU), - QU(12500579240208524895LLU), QU( 4484390097311355324LLU), - QU(15145801330700623870LLU), QU( 8055827661392944028LLU), - QU( 5865092976832712268LLU), QU(15159212508053625143LLU), - QU( 3560964582876483341LLU), QU( 4070052741344438280LLU), - QU( 6032585709886855634LLU), QU(15643262320904604873LLU), - QU( 2565119772293371111LLU), QU( 318314293065348260LLU), - QU(15047458749141511872LLU), QU( 7772788389811528730LLU), - QU( 7081187494343801976LLU), QU( 6465136009467253947LLU), - QU(10425940692543362069LLU), QU( 554608190318339115LLU), - QU(14796699860302125214LLU), QU( 1638153134431111443LLU), - QU(10336967447052276248LLU), QU( 8412308070396592958LLU), - QU( 4004557277152051226LLU), QU( 8143598997278774834LLU), - QU(16413323996508783221LLU), QU(13139418758033994949LLU), - QU( 9772709138335006667LLU), QU( 2818167159287157659LLU), - QU(17091740573832523669LLU), QU(14629199013130751608LLU), - QU(18268322711500338185LLU), QU( 8290963415675493063LLU), - QU( 8830864907452542588LLU), QU( 1614839084637494849LLU), - QU(14855358500870422231LLU), QU( 3472996748392519937LLU), - QU(15317151166268877716LLU), QU( 5825895018698400362LLU), - QU(16730208429367544129LLU), QU(10481156578141202800LLU), - QU( 4746166512382823750LLU), QU(12720876014472464998LLU), - QU( 8825177124486735972LLU), QU(13733447296837467838LLU), - QU( 6412293741681359625LLU), QU( 8313213138756135033LLU), - QU(11421481194803712517LLU), QU( 7997007691544174032LLU), - QU( 6812963847917605930LLU), QU( 9683091901227558641LLU), - QU(14703594165860324713LLU), QU( 1775476144519618309LLU), - QU( 2724283288516469519LLU), QU( 717642555185856868LLU), - QU( 8736402192215092346LLU), QU(11878800336431381021LLU), - QU( 4348816066017061293LLU), QU( 6115112756583631307LLU), - QU( 9176597239667142976LLU), QU(12615622714894259204LLU), - QU(10283406711301385987LLU), QU( 5111762509485379420LLU), - QU( 3118290051198688449LLU), QU( 7345123071632232145LLU), - QU( 9176423451688682359LLU), QU( 4843865456157868971LLU), - QU(12008036363752566088LLU), QU(12058837181919397720LLU), - QU( 2145073958457347366LLU), QU( 1526504881672818067LLU), - QU( 3488830105567134848LLU), QU(13208362960674805143LLU), - QU( 4077549672899572192LLU), QU( 7770995684693818365LLU), - QU( 1398532341546313593LLU), QU(12711859908703927840LLU), - QU( 1417561172594446813LLU), QU(17045191024194170604LLU), - QU( 4101933177604931713LLU), QU(14708428834203480320LLU), - QU(17447509264469407724LLU), QU(14314821973983434255LLU), - QU(17990472271061617265LLU), QU( 5087756685841673942LLU), - QU(12797820586893859939LLU), QU( 1778128952671092879LLU), - QU( 3535918530508665898LLU), QU( 9035729701042481301LLU), - QU(14808661568277079962LLU), QU(14587345077537747914LLU), - QU(11920080002323122708LLU), QU( 6426515805197278753LLU), - QU( 3295612216725984831LLU), QU(11040722532100876120LLU), - QU(12305952936387598754LLU), QU(16097391899742004253LLU), - QU( 4908537335606182208LLU), QU(12446674552196795504LLU), - QU(16010497855816895177LLU), QU( 9194378874788615551LLU), - QU( 3382957529567613384LLU), QU( 5154647600754974077LLU), - QU( 9801822865328396141LLU), QU( 9023662173919288143LLU), - QU(17623115353825147868LLU), QU( 8238115767443015816LLU), - QU(15811444159859002560LLU), QU( 9085612528904059661LLU), - QU( 6888601089398614254LLU), QU( 258252992894160189LLU), - QU( 6704363880792428622LLU), QU( 6114966032147235763LLU), - QU(11075393882690261875LLU), QU( 8797664238933620407LLU), - QU( 5901892006476726920LLU), QU( 5309780159285518958LLU), - QU(14940808387240817367LLU), QU(14642032021449656698LLU), - QU( 9808256672068504139LLU), QU( 3670135111380607658LLU), - QU(11211211097845960152LLU), QU( 1474304506716695808LLU), - QU(15843166204506876239LLU), QU( 7661051252471780561LLU), - QU(10170905502249418476LLU), QU( 7801416045582028589LLU), - QU( 2763981484737053050LLU), QU( 9491377905499253054LLU), - QU(16201395896336915095LLU), QU( 9256513756442782198LLU), - QU( 5411283157972456034LLU), QU( 5059433122288321676LLU), - QU( 4327408006721123357LLU), QU( 9278544078834433377LLU), - QU( 7601527110882281612LLU), QU(11848295896975505251LLU), - QU(12096998801094735560LLU), QU(14773480339823506413LLU), - QU(15586227433895802149LLU), QU(12786541257830242872LLU), - QU( 6904692985140503067LLU), QU( 5309011515263103959LLU), - QU(12105257191179371066LLU), QU(14654380212442225037LLU), - QU( 2556774974190695009LLU), QU( 4461297399927600261LLU), - QU(14888225660915118646LLU), QU(14915459341148291824LLU), - QU( 2738802166252327631LLU), QU( 6047155789239131512LLU), - QU(12920545353217010338LLU), QU(10697617257007840205LLU), - QU( 2751585253158203504LLU), QU(13252729159780047496LLU), - QU(14700326134672815469LLU), QU(14082527904374600529LLU), - QU(16852962273496542070LLU), QU(17446675504235853907LLU), - QU(15019600398527572311LLU), QU(12312781346344081551LLU), - QU(14524667935039810450LLU), QU( 5634005663377195738LLU), - QU(11375574739525000569LLU), QU( 2423665396433260040LLU), - QU( 5222836914796015410LLU), QU( 4397666386492647387LLU), - QU( 4619294441691707638LLU), QU( 665088602354770716LLU), - QU(13246495665281593610LLU), QU( 6564144270549729409LLU), - QU(10223216188145661688LLU), QU( 3961556907299230585LLU), - QU(11543262515492439914LLU), QU(16118031437285993790LLU), - QU( 7143417964520166465LLU), QU(13295053515909486772LLU), - QU( 40434666004899675LLU), QU(17127804194038347164LLU), - QU( 8599165966560586269LLU), QU( 8214016749011284903LLU), - QU(13725130352140465239LLU), QU( 5467254474431726291LLU), - QU( 7748584297438219877LLU), QU(16933551114829772472LLU), - QU( 2169618439506799400LLU), QU( 2169787627665113463LLU), - QU(17314493571267943764LLU), QU(18053575102911354912LLU), - QU(11928303275378476973LLU), QU(11593850925061715550LLU), - QU(17782269923473589362LLU), QU( 3280235307704747039LLU), - QU( 6145343578598685149LLU), QU(17080117031114086090LLU), - QU(18066839902983594755LLU), QU( 6517508430331020706LLU), - QU( 8092908893950411541LLU), QU(12558378233386153732LLU), - QU( 4476532167973132976LLU), QU(16081642430367025016LLU), - QU( 4233154094369139361LLU), QU( 8693630486693161027LLU), - QU(11244959343027742285LLU), QU(12273503967768513508LLU), - QU(14108978636385284876LLU), QU( 7242414665378826984LLU), - QU( 6561316938846562432LLU), QU( 8601038474994665795LLU), - QU(17532942353612365904LLU), QU(17940076637020912186LLU), - QU( 7340260368823171304LLU), QU( 7061807613916067905LLU), - QU(10561734935039519326LLU), QU(17990796503724650862LLU), - QU( 6208732943911827159LLU), QU( 359077562804090617LLU), - QU(14177751537784403113LLU), QU(10659599444915362902LLU), - QU(15081727220615085833LLU), QU(13417573895659757486LLU), - QU(15513842342017811524LLU), QU(11814141516204288231LLU), - QU( 1827312513875101814LLU), QU( 2804611699894603103LLU), - QU(17116500469975602763LLU), QU(12270191815211952087LLU), - QU(12256358467786024988LLU), QU(18435021722453971267LLU), - QU( 671330264390865618LLU), QU( 476504300460286050LLU), - QU(16465470901027093441LLU), QU( 4047724406247136402LLU), - QU( 1322305451411883346LLU), QU( 1388308688834322280LLU), - QU( 7303989085269758176LLU), QU( 9323792664765233642LLU), - QU( 4542762575316368936LLU), QU(17342696132794337618LLU), - QU( 4588025054768498379LLU), QU(13415475057390330804LLU), - QU(17880279491733405570LLU), QU(10610553400618620353LLU), - QU( 3180842072658960139LLU), QU(13002966655454270120LLU), - QU( 1665301181064982826LLU), QU( 7083673946791258979LLU), - QU( 190522247122496820LLU), QU(17388280237250677740LLU), - QU( 8430770379923642945LLU), QU(12987180971921668584LLU), - QU( 2311086108365390642LLU), QU( 2870984383579822345LLU), - QU(14014682609164653318LLU), QU(14467187293062251484LLU), - QU( 192186361147413298LLU), QU(15171951713531796524LLU), - QU( 9900305495015948728LLU), QU(17958004775615466344LLU), - QU(14346380954498606514LLU), QU(18040047357617407096LLU), - QU( 5035237584833424532LLU), QU(15089555460613972287LLU), - QU( 4131411873749729831LLU), QU( 1329013581168250330LLU), - QU(10095353333051193949LLU), QU(10749518561022462716LLU), - QU( 9050611429810755847LLU), QU(15022028840236655649LLU), - QU( 8775554279239748298LLU), QU(13105754025489230502LLU), - QU(15471300118574167585LLU), QU( 89864764002355628LLU), - QU( 8776416323420466637LLU), QU( 5280258630612040891LLU), - QU( 2719174488591862912LLU), QU( 7599309137399661994LLU), - QU(15012887256778039979LLU), QU(14062981725630928925LLU), - QU(12038536286991689603LLU), QU( 7089756544681775245LLU), - QU(10376661532744718039LLU), QU( 1265198725901533130LLU), - QU(13807996727081142408LLU), QU( 2935019626765036403LLU), - QU( 7651672460680700141LLU), QU( 3644093016200370795LLU), - QU( 2840982578090080674LLU), QU(17956262740157449201LLU), - QU(18267979450492880548LLU), QU(11799503659796848070LLU), - QU( 9942537025669672388LLU), QU(11886606816406990297LLU), - QU( 5488594946437447576LLU), QU( 7226714353282744302LLU), - QU( 3784851653123877043LLU), QU( 878018453244803041LLU), - QU(12110022586268616085LLU), QU( 734072179404675123LLU), - QU(11869573627998248542LLU), QU( 469150421297783998LLU), - QU( 260151124912803804LLU), QU(11639179410120968649LLU), - QU( 9318165193840846253LLU), QU(12795671722734758075LLU), - QU(15318410297267253933LLU), QU( 691524703570062620LLU), - QU( 5837129010576994601LLU), QU(15045963859726941052LLU), - QU( 5850056944932238169LLU), QU(12017434144750943807LLU), - QU( 7447139064928956574LLU), QU( 3101711812658245019LLU), - QU(16052940704474982954LLU), QU(18195745945986994042LLU), - QU( 8932252132785575659LLU), QU(13390817488106794834LLU), - QU(11582771836502517453LLU), QU( 4964411326683611686LLU), - QU( 2195093981702694011LLU), QU(14145229538389675669LLU), - QU(16459605532062271798LLU), QU( 866316924816482864LLU), - QU( 4593041209937286377LLU), QU( 8415491391910972138LLU), - QU( 4171236715600528969LLU), QU(16637569303336782889LLU), - QU( 2002011073439212680LLU), QU(17695124661097601411LLU), - QU( 4627687053598611702LLU), QU( 7895831936020190403LLU), - QU( 8455951300917267802LLU), QU( 2923861649108534854LLU), - QU( 8344557563927786255LLU), QU( 6408671940373352556LLU), - QU(12210227354536675772LLU), QU(14294804157294222295LLU), - QU(10103022425071085127LLU), QU(10092959489504123771LLU), - QU( 6554774405376736268LLU), QU(12629917718410641774LLU), - QU( 6260933257596067126LLU), QU( 2460827021439369673LLU), - QU( 2541962996717103668LLU), QU( 597377203127351475LLU), - QU( 5316984203117315309LLU), QU( 4811211393563241961LLU), - QU(13119698597255811641LLU), QU( 8048691512862388981LLU), - QU(10216818971194073842LLU), QU( 4612229970165291764LLU), - QU(10000980798419974770LLU), QU( 6877640812402540687LLU), - QU( 1488727563290436992LLU), QU( 2227774069895697318LLU), - QU(11237754507523316593LLU), QU(13478948605382290972LLU), - QU( 1963583846976858124LLU), QU( 5512309205269276457LLU), - QU( 3972770164717652347LLU), QU( 3841751276198975037LLU), - QU(10283343042181903117LLU), QU( 8564001259792872199LLU), - QU(16472187244722489221LLU), QU( 8953493499268945921LLU), - QU( 3518747340357279580LLU), QU( 4003157546223963073LLU), - QU( 3270305958289814590LLU), QU( 3966704458129482496LLU), - QU( 8122141865926661939LLU), QU(14627734748099506653LLU), - QU(13064426990862560568LLU), QU( 2414079187889870829LLU), - QU( 5378461209354225306LLU), QU(10841985740128255566LLU), - QU( 538582442885401738LLU), QU( 7535089183482905946LLU), - QU(16117559957598879095LLU), QU( 8477890721414539741LLU), - QU( 1459127491209533386LLU), QU(17035126360733620462LLU), - QU( 8517668552872379126LLU), QU(10292151468337355014LLU), - QU(17081267732745344157LLU), QU(13751455337946087178LLU), - QU(14026945459523832966LLU), QU( 6653278775061723516LLU), - QU(10619085543856390441LLU), QU( 2196343631481122885LLU), - QU(10045966074702826136LLU), QU(10082317330452718282LLU), - QU( 5920859259504831242LLU), QU( 9951879073426540617LLU), - QU( 7074696649151414158LLU), QU(15808193543879464318LLU), - QU( 7385247772746953374LLU), QU( 3192003544283864292LLU), - QU(18153684490917593847LLU), QU(12423498260668568905LLU), - QU(10957758099756378169LLU), QU(11488762179911016040LLU), - QU( 2099931186465333782LLU), QU(11180979581250294432LLU), - QU( 8098916250668367933LLU), QU( 3529200436790763465LLU), - QU(12988418908674681745LLU), QU( 6147567275954808580LLU), - QU( 3207503344604030989LLU), QU(10761592604898615360LLU), - QU( 229854861031893504LLU), QU( 8809853962667144291LLU), - QU(13957364469005693860LLU), QU( 7634287665224495886LLU), - QU(12353487366976556874LLU), QU( 1134423796317152034LLU), - QU( 2088992471334107068LLU), QU( 7393372127190799698LLU), - QU( 1845367839871058391LLU), QU( 207922563987322884LLU), - QU(11960870813159944976LLU), QU(12182120053317317363LLU), - QU(17307358132571709283LLU), QU(13871081155552824936LLU), - QU(18304446751741566262LLU), QU( 7178705220184302849LLU), - QU(10929605677758824425LLU), QU(16446976977835806844LLU), - QU(13723874412159769044LLU), QU( 6942854352100915216LLU), - QU( 1726308474365729390LLU), QU( 2150078766445323155LLU), - QU(15345558947919656626LLU), QU(12145453828874527201LLU), - QU( 2054448620739726849LLU), QU( 2740102003352628137LLU), - QU(11294462163577610655LLU), QU( 756164283387413743LLU), - QU(17841144758438810880LLU), QU(10802406021185415861LLU), - QU( 8716455530476737846LLU), QU( 6321788834517649606LLU), - QU(14681322910577468426LLU), QU(17330043563884336387LLU), - QU(12701802180050071614LLU), QU(14695105111079727151LLU), - QU( 5112098511654172830LLU), QU( 4957505496794139973LLU), - QU( 8270979451952045982LLU), QU(12307685939199120969LLU), - QU(12425799408953443032LLU), QU( 8376410143634796588LLU), - QU(16621778679680060464LLU), QU( 3580497854566660073LLU), - QU( 1122515747803382416LLU), QU( 857664980960597599LLU), - QU( 6343640119895925918LLU), QU(12878473260854462891LLU), - QU(10036813920765722626LLU), QU(14451335468363173812LLU), - QU( 5476809692401102807LLU), QU(16442255173514366342LLU), - QU(13060203194757167104LLU), QU(14354124071243177715LLU), - QU(15961249405696125227LLU), QU(13703893649690872584LLU), - QU( 363907326340340064LLU), QU( 6247455540491754842LLU), - QU(12242249332757832361LLU), QU( 156065475679796717LLU), - QU( 9351116235749732355LLU), QU( 4590350628677701405LLU), - QU( 1671195940982350389LLU), QU(13501398458898451905LLU), - QU( 6526341991225002255LLU), QU( 1689782913778157592LLU), - QU( 7439222350869010334LLU), QU(13975150263226478308LLU), - QU(11411961169932682710LLU), QU(17204271834833847277LLU), - QU( 541534742544435367LLU), QU( 6591191931218949684LLU), - QU( 2645454775478232486LLU), QU( 4322857481256485321LLU), - QU( 8477416487553065110LLU), QU(12902505428548435048LLU), - QU( 971445777981341415LLU), QU(14995104682744976712LLU), - QU( 4243341648807158063LLU), QU( 8695061252721927661LLU), - QU( 5028202003270177222LLU), QU( 2289257340915567840LLU), - QU(13870416345121866007LLU), QU(13994481698072092233LLU), - QU( 6912785400753196481LLU), QU( 2278309315841980139LLU), - QU( 4329765449648304839LLU), QU( 5963108095785485298LLU), - QU( 4880024847478722478LLU), QU(16015608779890240947LLU), - QU( 1866679034261393544LLU), QU( 914821179919731519LLU), - QU( 9643404035648760131LLU), QU( 2418114953615593915LLU), - QU( 944756836073702374LLU), QU(15186388048737296834LLU), - QU( 7723355336128442206LLU), QU( 7500747479679599691LLU), - QU(18013961306453293634LLU), QU( 2315274808095756456LLU), - QU(13655308255424029566LLU), QU(17203800273561677098LLU), - QU( 1382158694422087756LLU), QU( 5090390250309588976LLU), - QU( 517170818384213989LLU), QU( 1612709252627729621LLU), - QU( 1330118955572449606LLU), QU( 300922478056709885LLU), - QU(18115693291289091987LLU), QU(13491407109725238321LLU), - QU(15293714633593827320LLU), QU( 5151539373053314504LLU), - QU( 5951523243743139207LLU), QU(14459112015249527975LLU), - QU( 5456113959000700739LLU), QU( 3877918438464873016LLU), - QU(12534071654260163555LLU), QU(15871678376893555041LLU), - QU(11005484805712025549LLU), QU(16353066973143374252LLU), - QU( 4358331472063256685LLU), QU( 8268349332210859288LLU), - QU(12485161590939658075LLU), QU(13955993592854471343LLU), - QU( 5911446886848367039LLU), QU(14925834086813706974LLU), - QU( 6590362597857994805LLU), QU( 1280544923533661875LLU), - QU( 1637756018947988164LLU), QU( 4734090064512686329LLU), - QU(16693705263131485912LLU), QU( 6834882340494360958LLU), - QU( 8120732176159658505LLU), QU( 2244371958905329346LLU), - QU(10447499707729734021LLU), QU( 7318742361446942194LLU), - QU( 8032857516355555296LLU), QU(14023605983059313116LLU), - QU( 1032336061815461376LLU), QU( 9840995337876562612LLU), - QU( 9869256223029203587LLU), QU(12227975697177267636LLU), - QU(12728115115844186033LLU), QU( 7752058479783205470LLU), - QU( 729733219713393087LLU), QU(12954017801239007622LLU) + QU(16924766246869039260ULL), QU( 8201438687333352714ULL), + QU( 2265290287015001750ULL), QU(18397264611805473832ULL), + QU( 3375255223302384358ULL), QU( 6345559975416828796ULL), + QU(18229739242790328073ULL), QU( 7596792742098800905ULL), + QU( 255338647169685981ULL), QU( 2052747240048610300ULL), + QU(18328151576097299343ULL), QU(12472905421133796567ULL), + QU(11315245349717600863ULL), QU(16594110197775871209ULL), + QU(15708751964632456450ULL), QU(10452031272054632535ULL), + QU(11097646720811454386ULL), QU( 4556090668445745441ULL), + QU(17116187693090663106ULL), QU(14931526836144510645ULL), + QU( 9190752218020552591ULL), QU( 9625800285771901401ULL), + QU(13995141077659972832ULL), QU( 5194209094927829625ULL), + QU( 4156788379151063303ULL), QU( 8523452593770139494ULL), + QU(14082382103049296727ULL), QU( 2462601863986088483ULL), + QU( 3030583461592840678ULL), QU( 5221622077872827681ULL), + QU( 3084210671228981236ULL), QU(13956758381389953823ULL), + QU(13503889856213423831ULL), QU(15696904024189836170ULL), + QU( 4612584152877036206ULL), QU( 6231135538447867881ULL), + QU(10172457294158869468ULL), QU( 6452258628466708150ULL), + QU(14044432824917330221ULL), QU( 370168364480044279ULL), + QU(10102144686427193359ULL), QU( 667870489994776076ULL), + QU( 2732271956925885858ULL), QU(18027788905977284151ULL), + QU(15009842788582923859ULL), QU( 7136357960180199542ULL), + QU(15901736243475578127ULL), QU(16951293785352615701ULL), + QU(10551492125243691632ULL), QU(17668869969146434804ULL), + QU(13646002971174390445ULL), QU( 9804471050759613248ULL), + QU( 5511670439655935493ULL), QU(18103342091070400926ULL), + QU(17224512747665137533ULL), QU(15534627482992618168ULL), + QU( 1423813266186582647ULL), QU(15821176807932930024ULL), + QU( 30323369733607156ULL), QU(11599382494723479403ULL), + QU( 653856076586810062ULL), QU( 3176437395144899659ULL), + QU(14028076268147963917ULL), QU(16156398271809666195ULL), + QU( 3166955484848201676ULL), QU( 5746805620136919390ULL), + QU(17297845208891256593ULL), QU(11691653183226428483ULL), + QU(17900026146506981577ULL), QU(15387382115755971042ULL), + QU(16923567681040845943ULL), QU( 8039057517199388606ULL), + QU(11748409241468629263ULL), QU( 794358245539076095ULL), + QU(13438501964693401242ULL), QU(14036803236515618962ULL), + QU( 5252311215205424721ULL), QU(17806589612915509081ULL), + QU( 6802767092397596006ULL), QU(14212120431184557140ULL), + QU( 1072951366761385712ULL), QU(13098491780722836296ULL), + QU( 9466676828710797353ULL), QU(12673056849042830081ULL), + QU(12763726623645357580ULL), QU(16468961652999309493ULL), + QU(15305979875636438926ULL), QU(17444713151223449734ULL), + QU( 5692214267627883674ULL), QU(13049589139196151505ULL), + QU( 880115207831670745ULL), QU( 1776529075789695498ULL), + QU(16695225897801466485ULL), QU(10666901778795346845ULL), + QU( 6164389346722833869ULL), QU( 2863817793264300475ULL), + QU( 9464049921886304754ULL), QU( 3993566636740015468ULL), + QU( 9983749692528514136ULL), QU(16375286075057755211ULL), + QU(16042643417005440820ULL), QU(11445419662923489877ULL), + QU( 7999038846885158836ULL), QU( 6721913661721511535ULL), + QU( 5363052654139357320ULL), QU( 1817788761173584205ULL), + QU(13290974386445856444ULL), QU( 4650350818937984680ULL), + QU( 8219183528102484836ULL), QU( 1569862923500819899ULL), + QU( 4189359732136641860ULL), QU(14202822961683148583ULL), + QU( 4457498315309429058ULL), QU(13089067387019074834ULL), + QU(11075517153328927293ULL), QU(10277016248336668389ULL), + QU( 7070509725324401122ULL), QU(17808892017780289380ULL), + QU(13143367339909287349ULL), QU( 1377743745360085151ULL), + QU( 5749341807421286485ULL), QU(14832814616770931325ULL), + QU( 7688820635324359492ULL), QU(10960474011539770045ULL), + QU( 81970066653179790ULL), QU(12619476072607878022ULL), + QU( 4419566616271201744ULL), QU(15147917311750568503ULL), + QU( 5549739182852706345ULL), QU( 7308198397975204770ULL), + QU(13580425496671289278ULL), QU(17070764785210130301ULL), + QU( 8202832846285604405ULL), QU( 6873046287640887249ULL), + QU( 6927424434308206114ULL), QU( 6139014645937224874ULL), + QU(10290373645978487639ULL), QU(15904261291701523804ULL), + QU( 9628743442057826883ULL), QU(18383429096255546714ULL), + QU( 4977413265753686967ULL), QU( 7714317492425012869ULL), + QU( 9025232586309926193ULL), QU(14627338359776709107ULL), + QU(14759849896467790763ULL), QU(10931129435864423252ULL), + QU( 4588456988775014359ULL), QU(10699388531797056724ULL), + QU( 468652268869238792ULL), QU( 5755943035328078086ULL), + QU( 2102437379988580216ULL), QU( 9986312786506674028ULL), + QU( 2654207180040945604ULL), QU( 8726634790559960062ULL), + QU( 100497234871808137ULL), QU( 2800137176951425819ULL), + QU( 6076627612918553487ULL), QU( 5780186919186152796ULL), + QU( 8179183595769929098ULL), QU( 6009426283716221169ULL), + QU( 2796662551397449358ULL), QU( 1756961367041986764ULL), + QU( 6972897917355606205ULL), QU(14524774345368968243ULL), + QU( 2773529684745706940ULL), QU( 4853632376213075959ULL), + QU( 4198177923731358102ULL), QU( 8271224913084139776ULL), + QU( 2741753121611092226ULL), QU(16782366145996731181ULL), + QU(15426125238972640790ULL), QU(13595497100671260342ULL), + QU( 3173531022836259898ULL), QU( 6573264560319511662ULL), + QU(18041111951511157441ULL), QU( 2351433581833135952ULL), + QU( 3113255578908173487ULL), QU( 1739371330877858784ULL), + QU(16046126562789165480ULL), QU( 8072101652214192925ULL), + QU(15267091584090664910ULL), QU( 9309579200403648940ULL), + QU( 5218892439752408722ULL), QU(14492477246004337115ULL), + QU(17431037586679770619ULL), QU( 7385248135963250480ULL), + QU( 9580144956565560660ULL), QU( 4919546228040008720ULL), + QU(15261542469145035584ULL), QU(18233297270822253102ULL), + QU( 5453248417992302857ULL), QU( 9309519155931460285ULL), + QU(10342813012345291756ULL), QU(15676085186784762381ULL), + QU(15912092950691300645ULL), QU( 9371053121499003195ULL), + QU( 9897186478226866746ULL), QU(14061858287188196327ULL), + QU( 122575971620788119ULL), QU(12146750969116317754ULL), + QU( 4438317272813245201ULL), QU( 8332576791009527119ULL), + QU(13907785691786542057ULL), QU(10374194887283287467ULL), + QU( 2098798755649059566ULL), QU( 3416235197748288894ULL), + QU( 8688269957320773484ULL), QU( 7503964602397371571ULL), + QU(16724977015147478236ULL), QU( 9461512855439858184ULL), + QU(13259049744534534727ULL), QU( 3583094952542899294ULL), + QU( 8764245731305528292ULL), QU(13240823595462088985ULL), + QU(13716141617617910448ULL), QU(18114969519935960955ULL), + QU( 2297553615798302206ULL), QU( 4585521442944663362ULL), + QU(17776858680630198686ULL), QU( 4685873229192163363ULL), + QU( 152558080671135627ULL), QU(15424900540842670088ULL), + QU(13229630297130024108ULL), QU(17530268788245718717ULL), + QU(16675633913065714144ULL), QU( 3158912717897568068ULL), + QU(15399132185380087288ULL), QU( 7401418744515677872ULL), + QU(13135412922344398535ULL), QU( 6385314346100509511ULL), + QU(13962867001134161139ULL), QU(10272780155442671999ULL), + QU(12894856086597769142ULL), QU(13340877795287554994ULL), + QU(12913630602094607396ULL), QU(12543167911119793857ULL), + QU(17343570372251873096ULL), QU(10959487764494150545ULL), + QU( 6966737953093821128ULL), QU(13780699135496988601ULL), + QU( 4405070719380142046ULL), QU(14923788365607284982ULL), + QU( 2869487678905148380ULL), QU( 6416272754197188403ULL), + QU(15017380475943612591ULL), QU( 1995636220918429487ULL), + QU( 3402016804620122716ULL), QU(15800188663407057080ULL), + QU(11362369990390932882ULL), QU(15262183501637986147ULL), + QU(10239175385387371494ULL), QU( 9352042420365748334ULL), + QU( 1682457034285119875ULL), QU( 1724710651376289644ULL), + QU( 2038157098893817966ULL), QU( 9897825558324608773ULL), + QU( 1477666236519164736ULL), QU(16835397314511233640ULL), + QU(10370866327005346508ULL), QU(10157504370660621982ULL), + QU(12113904045335882069ULL), QU(13326444439742783008ULL), + QU(11302769043000765804ULL), QU(13594979923955228484ULL), + QU(11779351762613475968ULL), QU( 3786101619539298383ULL), + QU( 8021122969180846063ULL), QU(15745904401162500495ULL), + QU(10762168465993897267ULL), QU(13552058957896319026ULL), + QU(11200228655252462013ULL), QU( 5035370357337441226ULL), + QU( 7593918984545500013ULL), QU( 5418554918361528700ULL), + QU( 4858270799405446371ULL), QU( 9974659566876282544ULL), + QU(18227595922273957859ULL), QU( 2772778443635656220ULL), + QU(14285143053182085385ULL), QU( 9939700992429600469ULL), + QU(12756185904545598068ULL), QU( 2020783375367345262ULL), + QU( 57026775058331227ULL), QU( 950827867930065454ULL), + QU( 6602279670145371217ULL), QU( 2291171535443566929ULL), + QU( 5832380724425010313ULL), QU( 1220343904715982285ULL), + QU(17045542598598037633ULL), QU(15460481779702820971ULL), + QU(13948388779949365130ULL), QU(13975040175430829518ULL), + QU(17477538238425541763ULL), QU(11104663041851745725ULL), + QU(15860992957141157587ULL), QU(14529434633012950138ULL), + QU( 2504838019075394203ULL), QU( 7512113882611121886ULL), + QU( 4859973559980886617ULL), QU( 1258601555703250219ULL), + QU(15594548157514316394ULL), QU( 4516730171963773048ULL), + QU(11380103193905031983ULL), QU( 6809282239982353344ULL), + QU(18045256930420065002ULL), QU( 2453702683108791859ULL), + QU( 977214582986981460ULL), QU( 2006410402232713466ULL), + QU( 6192236267216378358ULL), QU( 3429468402195675253ULL), + QU(18146933153017348921ULL), QU(17369978576367231139ULL), + QU( 1246940717230386603ULL), QU(11335758870083327110ULL), + QU(14166488801730353682ULL), QU( 9008573127269635732ULL), + QU(10776025389820643815ULL), QU(15087605441903942962ULL), + QU( 1359542462712147922ULL), QU(13898874411226454206ULL), + QU(17911176066536804411ULL), QU( 9435590428600085274ULL), + QU( 294488509967864007ULL), QU( 8890111397567922046ULL), + QU( 7987823476034328778ULL), QU(13263827582440967651ULL), + QU( 7503774813106751573ULL), QU(14974747296185646837ULL), + QU( 8504765037032103375ULL), QU(17340303357444536213ULL), + QU( 7704610912964485743ULL), QU( 8107533670327205061ULL), + QU( 9062969835083315985ULL), QU(16968963142126734184ULL), + QU(12958041214190810180ULL), QU( 2720170147759570200ULL), + QU( 2986358963942189566ULL), QU(14884226322219356580ULL), + QU( 286224325144368520ULL), QU(11313800433154279797ULL), + QU(18366849528439673248ULL), QU(17899725929482368789ULL), + QU( 3730004284609106799ULL), QU( 1654474302052767205ULL), + QU( 5006698007047077032ULL), QU( 8196893913601182838ULL), + QU(15214541774425211640ULL), QU(17391346045606626073ULL), + QU( 8369003584076969089ULL), QU( 3939046733368550293ULL), + QU(10178639720308707785ULL), QU( 2180248669304388697ULL), + QU( 62894391300126322ULL), QU( 9205708961736223191ULL), + QU( 6837431058165360438ULL), QU( 3150743890848308214ULL), + QU(17849330658111464583ULL), QU(12214815643135450865ULL), + QU(13410713840519603402ULL), QU( 3200778126692046802ULL), + QU(13354780043041779313ULL), QU( 800850022756886036ULL), + QU(15660052933953067433ULL), QU( 6572823544154375676ULL), + QU(11030281857015819266ULL), QU(12682241941471433835ULL), + QU(11654136407300274693ULL), QU( 4517795492388641109ULL), + QU( 9757017371504524244ULL), QU(17833043400781889277ULL), + QU(12685085201747792227ULL), QU(10408057728835019573ULL), + QU( 98370418513455221ULL), QU( 6732663555696848598ULL), + QU(13248530959948529780ULL), QU( 3530441401230622826ULL), + QU(18188251992895660615ULL), QU( 1847918354186383756ULL), + QU( 1127392190402660921ULL), QU(11293734643143819463ULL), + QU( 3015506344578682982ULL), QU(13852645444071153329ULL), + QU( 2121359659091349142ULL), QU( 1294604376116677694ULL), + QU( 5616576231286352318ULL), QU( 7112502442954235625ULL), + QU(11676228199551561689ULL), QU(12925182803007305359ULL), + QU( 7852375518160493082ULL), QU( 1136513130539296154ULL), + QU( 5636923900916593195ULL), QU( 3221077517612607747ULL), + QU(17784790465798152513ULL), QU( 3554210049056995938ULL), + QU(17476839685878225874ULL), QU( 3206836372585575732ULL), + QU( 2765333945644823430ULL), QU(10080070903718799528ULL), + QU( 5412370818878286353ULL), QU( 9689685887726257728ULL), + QU( 8236117509123533998ULL), QU( 1951139137165040214ULL), + QU( 4492205209227980349ULL), QU(16541291230861602967ULL), + QU( 1424371548301437940ULL), QU( 9117562079669206794ULL), + QU(14374681563251691625ULL), QU(13873164030199921303ULL), + QU( 6680317946770936731ULL), QU(15586334026918276214ULL), + QU(10896213950976109802ULL), QU( 9506261949596413689ULL), + QU( 9903949574308040616ULL), QU( 6038397344557204470ULL), + QU( 174601465422373648ULL), QU(15946141191338238030ULL), + QU(17142225620992044937ULL), QU( 7552030283784477064ULL), + QU( 2947372384532947997ULL), QU( 510797021688197711ULL), + QU( 4962499439249363461ULL), QU( 23770320158385357ULL), + QU( 959774499105138124ULL), QU( 1468396011518788276ULL), + QU( 2015698006852312308ULL), QU( 4149400718489980136ULL), + QU( 5992916099522371188ULL), QU(10819182935265531076ULL), + QU(16189787999192351131ULL), QU( 342833961790261950ULL), + QU(12470830319550495336ULL), QU(18128495041912812501ULL), + QU( 1193600899723524337ULL), QU( 9056793666590079770ULL), + QU( 2154021227041669041ULL), QU( 4963570213951235735ULL), + QU( 4865075960209211409ULL), QU( 2097724599039942963ULL), + QU( 2024080278583179845ULL), QU(11527054549196576736ULL), + QU(10650256084182390252ULL), QU( 4808408648695766755ULL), + QU( 1642839215013788844ULL), QU(10607187948250398390ULL), + QU( 7076868166085913508ULL), QU( 730522571106887032ULL), + QU(12500579240208524895ULL), QU( 4484390097311355324ULL), + QU(15145801330700623870ULL), QU( 8055827661392944028ULL), + QU( 5865092976832712268ULL), QU(15159212508053625143ULL), + QU( 3560964582876483341ULL), QU( 4070052741344438280ULL), + QU( 6032585709886855634ULL), QU(15643262320904604873ULL), + QU( 2565119772293371111ULL), QU( 318314293065348260ULL), + QU(15047458749141511872ULL), QU( 7772788389811528730ULL), + QU( 7081187494343801976ULL), QU( 6465136009467253947ULL), + QU(10425940692543362069ULL), QU( 554608190318339115ULL), + QU(14796699860302125214ULL), QU( 1638153134431111443ULL), + QU(10336967447052276248ULL), QU( 8412308070396592958ULL), + QU( 4004557277152051226ULL), QU( 8143598997278774834ULL), + QU(16413323996508783221ULL), QU(13139418758033994949ULL), + QU( 9772709138335006667ULL), QU( 2818167159287157659ULL), + QU(17091740573832523669ULL), QU(14629199013130751608ULL), + QU(18268322711500338185ULL), QU( 8290963415675493063ULL), + QU( 8830864907452542588ULL), QU( 1614839084637494849ULL), + QU(14855358500870422231ULL), QU( 3472996748392519937ULL), + QU(15317151166268877716ULL), QU( 5825895018698400362ULL), + QU(16730208429367544129ULL), QU(10481156578141202800ULL), + QU( 4746166512382823750ULL), QU(12720876014472464998ULL), + QU( 8825177124486735972ULL), QU(13733447296837467838ULL), + QU( 6412293741681359625ULL), QU( 8313213138756135033ULL), + QU(11421481194803712517ULL), QU( 7997007691544174032ULL), + QU( 6812963847917605930ULL), QU( 9683091901227558641ULL), + QU(14703594165860324713ULL), QU( 1775476144519618309ULL), + QU( 2724283288516469519ULL), QU( 717642555185856868ULL), + QU( 8736402192215092346ULL), QU(11878800336431381021ULL), + QU( 4348816066017061293ULL), QU( 6115112756583631307ULL), + QU( 9176597239667142976ULL), QU(12615622714894259204ULL), + QU(10283406711301385987ULL), QU( 5111762509485379420ULL), + QU( 3118290051198688449ULL), QU( 7345123071632232145ULL), + QU( 9176423451688682359ULL), QU( 4843865456157868971ULL), + QU(12008036363752566088ULL), QU(12058837181919397720ULL), + QU( 2145073958457347366ULL), QU( 1526504881672818067ULL), + QU( 3488830105567134848ULL), QU(13208362960674805143ULL), + QU( 4077549672899572192ULL), QU( 7770995684693818365ULL), + QU( 1398532341546313593ULL), QU(12711859908703927840ULL), + QU( 1417561172594446813ULL), QU(17045191024194170604ULL), + QU( 4101933177604931713ULL), QU(14708428834203480320ULL), + QU(17447509264469407724ULL), QU(14314821973983434255ULL), + QU(17990472271061617265ULL), QU( 5087756685841673942ULL), + QU(12797820586893859939ULL), QU( 1778128952671092879ULL), + QU( 3535918530508665898ULL), QU( 9035729701042481301ULL), + QU(14808661568277079962ULL), QU(14587345077537747914ULL), + QU(11920080002323122708ULL), QU( 6426515805197278753ULL), + QU( 3295612216725984831ULL), QU(11040722532100876120ULL), + QU(12305952936387598754ULL), QU(16097391899742004253ULL), + QU( 4908537335606182208ULL), QU(12446674552196795504ULL), + QU(16010497855816895177ULL), QU( 9194378874788615551ULL), + QU( 3382957529567613384ULL), QU( 5154647600754974077ULL), + QU( 9801822865328396141ULL), QU( 9023662173919288143ULL), + QU(17623115353825147868ULL), QU( 8238115767443015816ULL), + QU(15811444159859002560ULL), QU( 9085612528904059661ULL), + QU( 6888601089398614254ULL), QU( 258252992894160189ULL), + QU( 6704363880792428622ULL), QU( 6114966032147235763ULL), + QU(11075393882690261875ULL), QU( 8797664238933620407ULL), + QU( 5901892006476726920ULL), QU( 5309780159285518958ULL), + QU(14940808387240817367ULL), QU(14642032021449656698ULL), + QU( 9808256672068504139ULL), QU( 3670135111380607658ULL), + QU(11211211097845960152ULL), QU( 1474304506716695808ULL), + QU(15843166204506876239ULL), QU( 7661051252471780561ULL), + QU(10170905502249418476ULL), QU( 7801416045582028589ULL), + QU( 2763981484737053050ULL), QU( 9491377905499253054ULL), + QU(16201395896336915095ULL), QU( 9256513756442782198ULL), + QU( 5411283157972456034ULL), QU( 5059433122288321676ULL), + QU( 4327408006721123357ULL), QU( 9278544078834433377ULL), + QU( 7601527110882281612ULL), QU(11848295896975505251ULL), + QU(12096998801094735560ULL), QU(14773480339823506413ULL), + QU(15586227433895802149ULL), QU(12786541257830242872ULL), + QU( 6904692985140503067ULL), QU( 5309011515263103959ULL), + QU(12105257191179371066ULL), QU(14654380212442225037ULL), + QU( 2556774974190695009ULL), QU( 4461297399927600261ULL), + QU(14888225660915118646ULL), QU(14915459341148291824ULL), + QU( 2738802166252327631ULL), QU( 6047155789239131512ULL), + QU(12920545353217010338ULL), QU(10697617257007840205ULL), + QU( 2751585253158203504ULL), QU(13252729159780047496ULL), + QU(14700326134672815469ULL), QU(14082527904374600529ULL), + QU(16852962273496542070ULL), QU(17446675504235853907ULL), + QU(15019600398527572311ULL), QU(12312781346344081551ULL), + QU(14524667935039810450ULL), QU( 5634005663377195738ULL), + QU(11375574739525000569ULL), QU( 2423665396433260040ULL), + QU( 5222836914796015410ULL), QU( 4397666386492647387ULL), + QU( 4619294441691707638ULL), QU( 665088602354770716ULL), + QU(13246495665281593610ULL), QU( 6564144270549729409ULL), + QU(10223216188145661688ULL), QU( 3961556907299230585ULL), + QU(11543262515492439914ULL), QU(16118031437285993790ULL), + QU( 7143417964520166465ULL), QU(13295053515909486772ULL), + QU( 40434666004899675ULL), QU(17127804194038347164ULL), + QU( 8599165966560586269ULL), QU( 8214016749011284903ULL), + QU(13725130352140465239ULL), QU( 5467254474431726291ULL), + QU( 7748584297438219877ULL), QU(16933551114829772472ULL), + QU( 2169618439506799400ULL), QU( 2169787627665113463ULL), + QU(17314493571267943764ULL), QU(18053575102911354912ULL), + QU(11928303275378476973ULL), QU(11593850925061715550ULL), + QU(17782269923473589362ULL), QU( 3280235307704747039ULL), + QU( 6145343578598685149ULL), QU(17080117031114086090ULL), + QU(18066839902983594755ULL), QU( 6517508430331020706ULL), + QU( 8092908893950411541ULL), QU(12558378233386153732ULL), + QU( 4476532167973132976ULL), QU(16081642430367025016ULL), + QU( 4233154094369139361ULL), QU( 8693630486693161027ULL), + QU(11244959343027742285ULL), QU(12273503967768513508ULL), + QU(14108978636385284876ULL), QU( 7242414665378826984ULL), + QU( 6561316938846562432ULL), QU( 8601038474994665795ULL), + QU(17532942353612365904ULL), QU(17940076637020912186ULL), + QU( 7340260368823171304ULL), QU( 7061807613916067905ULL), + QU(10561734935039519326ULL), QU(17990796503724650862ULL), + QU( 6208732943911827159ULL), QU( 359077562804090617ULL), + QU(14177751537784403113ULL), QU(10659599444915362902ULL), + QU(15081727220615085833ULL), QU(13417573895659757486ULL), + QU(15513842342017811524ULL), QU(11814141516204288231ULL), + QU( 1827312513875101814ULL), QU( 2804611699894603103ULL), + QU(17116500469975602763ULL), QU(12270191815211952087ULL), + QU(12256358467786024988ULL), QU(18435021722453971267ULL), + QU( 671330264390865618ULL), QU( 476504300460286050ULL), + QU(16465470901027093441ULL), QU( 4047724406247136402ULL), + QU( 1322305451411883346ULL), QU( 1388308688834322280ULL), + QU( 7303989085269758176ULL), QU( 9323792664765233642ULL), + QU( 4542762575316368936ULL), QU(17342696132794337618ULL), + QU( 4588025054768498379ULL), QU(13415475057390330804ULL), + QU(17880279491733405570ULL), QU(10610553400618620353ULL), + QU( 3180842072658960139ULL), QU(13002966655454270120ULL), + QU( 1665301181064982826ULL), QU( 7083673946791258979ULL), + QU( 190522247122496820ULL), QU(17388280237250677740ULL), + QU( 8430770379923642945ULL), QU(12987180971921668584ULL), + QU( 2311086108365390642ULL), QU( 2870984383579822345ULL), + QU(14014682609164653318ULL), QU(14467187293062251484ULL), + QU( 192186361147413298ULL), QU(15171951713531796524ULL), + QU( 9900305495015948728ULL), QU(17958004775615466344ULL), + QU(14346380954498606514ULL), QU(18040047357617407096ULL), + QU( 5035237584833424532ULL), QU(15089555460613972287ULL), + QU( 4131411873749729831ULL), QU( 1329013581168250330ULL), + QU(10095353333051193949ULL), QU(10749518561022462716ULL), + QU( 9050611429810755847ULL), QU(15022028840236655649ULL), + QU( 8775554279239748298ULL), QU(13105754025489230502ULL), + QU(15471300118574167585ULL), QU( 89864764002355628ULL), + QU( 8776416323420466637ULL), QU( 5280258630612040891ULL), + QU( 2719174488591862912ULL), QU( 7599309137399661994ULL), + QU(15012887256778039979ULL), QU(14062981725630928925ULL), + QU(12038536286991689603ULL), QU( 7089756544681775245ULL), + QU(10376661532744718039ULL), QU( 1265198725901533130ULL), + QU(13807996727081142408ULL), QU( 2935019626765036403ULL), + QU( 7651672460680700141ULL), QU( 3644093016200370795ULL), + QU( 2840982578090080674ULL), QU(17956262740157449201ULL), + QU(18267979450492880548ULL), QU(11799503659796848070ULL), + QU( 9942537025669672388ULL), QU(11886606816406990297ULL), + QU( 5488594946437447576ULL), QU( 7226714353282744302ULL), + QU( 3784851653123877043ULL), QU( 878018453244803041ULL), + QU(12110022586268616085ULL), QU( 734072179404675123ULL), + QU(11869573627998248542ULL), QU( 469150421297783998ULL), + QU( 260151124912803804ULL), QU(11639179410120968649ULL), + QU( 9318165193840846253ULL), QU(12795671722734758075ULL), + QU(15318410297267253933ULL), QU( 691524703570062620ULL), + QU( 5837129010576994601ULL), QU(15045963859726941052ULL), + QU( 5850056944932238169ULL), QU(12017434144750943807ULL), + QU( 7447139064928956574ULL), QU( 3101711812658245019ULL), + QU(16052940704474982954ULL), QU(18195745945986994042ULL), + QU( 8932252132785575659ULL), QU(13390817488106794834ULL), + QU(11582771836502517453ULL), QU( 4964411326683611686ULL), + QU( 2195093981702694011ULL), QU(14145229538389675669ULL), + QU(16459605532062271798ULL), QU( 866316924816482864ULL), + QU( 4593041209937286377ULL), QU( 8415491391910972138ULL), + QU( 4171236715600528969ULL), QU(16637569303336782889ULL), + QU( 2002011073439212680ULL), QU(17695124661097601411ULL), + QU( 4627687053598611702ULL), QU( 7895831936020190403ULL), + QU( 8455951300917267802ULL), QU( 2923861649108534854ULL), + QU( 8344557563927786255ULL), QU( 6408671940373352556ULL), + QU(12210227354536675772ULL), QU(14294804157294222295ULL), + QU(10103022425071085127ULL), QU(10092959489504123771ULL), + QU( 6554774405376736268ULL), QU(12629917718410641774ULL), + QU( 6260933257596067126ULL), QU( 2460827021439369673ULL), + QU( 2541962996717103668ULL), QU( 597377203127351475ULL), + QU( 5316984203117315309ULL), QU( 4811211393563241961ULL), + QU(13119698597255811641ULL), QU( 8048691512862388981ULL), + QU(10216818971194073842ULL), QU( 4612229970165291764ULL), + QU(10000980798419974770ULL), QU( 6877640812402540687ULL), + QU( 1488727563290436992ULL), QU( 2227774069895697318ULL), + QU(11237754507523316593ULL), QU(13478948605382290972ULL), + QU( 1963583846976858124ULL), QU( 5512309205269276457ULL), + QU( 3972770164717652347ULL), QU( 3841751276198975037ULL), + QU(10283343042181903117ULL), QU( 8564001259792872199ULL), + QU(16472187244722489221ULL), QU( 8953493499268945921ULL), + QU( 3518747340357279580ULL), QU( 4003157546223963073ULL), + QU( 3270305958289814590ULL), QU( 3966704458129482496ULL), + QU( 8122141865926661939ULL), QU(14627734748099506653ULL), + QU(13064426990862560568ULL), QU( 2414079187889870829ULL), + QU( 5378461209354225306ULL), QU(10841985740128255566ULL), + QU( 538582442885401738ULL), QU( 7535089183482905946ULL), + QU(16117559957598879095ULL), QU( 8477890721414539741ULL), + QU( 1459127491209533386ULL), QU(17035126360733620462ULL), + QU( 8517668552872379126ULL), QU(10292151468337355014ULL), + QU(17081267732745344157ULL), QU(13751455337946087178ULL), + QU(14026945459523832966ULL), QU( 6653278775061723516ULL), + QU(10619085543856390441ULL), QU( 2196343631481122885ULL), + QU(10045966074702826136ULL), QU(10082317330452718282ULL), + QU( 5920859259504831242ULL), QU( 9951879073426540617ULL), + QU( 7074696649151414158ULL), QU(15808193543879464318ULL), + QU( 7385247772746953374ULL), QU( 3192003544283864292ULL), + QU(18153684490917593847ULL), QU(12423498260668568905ULL), + QU(10957758099756378169ULL), QU(11488762179911016040ULL), + QU( 2099931186465333782ULL), QU(11180979581250294432ULL), + QU( 8098916250668367933ULL), QU( 3529200436790763465ULL), + QU(12988418908674681745ULL), QU( 6147567275954808580ULL), + QU( 3207503344604030989ULL), QU(10761592604898615360ULL), + QU( 229854861031893504ULL), QU( 8809853962667144291ULL), + QU(13957364469005693860ULL), QU( 7634287665224495886ULL), + QU(12353487366976556874ULL), QU( 1134423796317152034ULL), + QU( 2088992471334107068ULL), QU( 7393372127190799698ULL), + QU( 1845367839871058391ULL), QU( 207922563987322884ULL), + QU(11960870813159944976ULL), QU(12182120053317317363ULL), + QU(17307358132571709283ULL), QU(13871081155552824936ULL), + QU(18304446751741566262ULL), QU( 7178705220184302849ULL), + QU(10929605677758824425ULL), QU(16446976977835806844ULL), + QU(13723874412159769044ULL), QU( 6942854352100915216ULL), + QU( 1726308474365729390ULL), QU( 2150078766445323155ULL), + QU(15345558947919656626ULL), QU(12145453828874527201ULL), + QU( 2054448620739726849ULL), QU( 2740102003352628137ULL), + QU(11294462163577610655ULL), QU( 756164283387413743ULL), + QU(17841144758438810880ULL), QU(10802406021185415861ULL), + QU( 8716455530476737846ULL), QU( 6321788834517649606ULL), + QU(14681322910577468426ULL), QU(17330043563884336387ULL), + QU(12701802180050071614ULL), QU(14695105111079727151ULL), + QU( 5112098511654172830ULL), QU( 4957505496794139973ULL), + QU( 8270979451952045982ULL), QU(12307685939199120969ULL), + QU(12425799408953443032ULL), QU( 8376410143634796588ULL), + QU(16621778679680060464ULL), QU( 3580497854566660073ULL), + QU( 1122515747803382416ULL), QU( 857664980960597599ULL), + QU( 6343640119895925918ULL), QU(12878473260854462891ULL), + QU(10036813920765722626ULL), QU(14451335468363173812ULL), + QU( 5476809692401102807ULL), QU(16442255173514366342ULL), + QU(13060203194757167104ULL), QU(14354124071243177715ULL), + QU(15961249405696125227ULL), QU(13703893649690872584ULL), + QU( 363907326340340064ULL), QU( 6247455540491754842ULL), + QU(12242249332757832361ULL), QU( 156065475679796717ULL), + QU( 9351116235749732355ULL), QU( 4590350628677701405ULL), + QU( 1671195940982350389ULL), QU(13501398458898451905ULL), + QU( 6526341991225002255ULL), QU( 1689782913778157592ULL), + QU( 7439222350869010334ULL), QU(13975150263226478308ULL), + QU(11411961169932682710ULL), QU(17204271834833847277ULL), + QU( 541534742544435367ULL), QU( 6591191931218949684ULL), + QU( 2645454775478232486ULL), QU( 4322857481256485321ULL), + QU( 8477416487553065110ULL), QU(12902505428548435048ULL), + QU( 971445777981341415ULL), QU(14995104682744976712ULL), + QU( 4243341648807158063ULL), QU( 8695061252721927661ULL), + QU( 5028202003270177222ULL), QU( 2289257340915567840ULL), + QU(13870416345121866007ULL), QU(13994481698072092233ULL), + QU( 6912785400753196481ULL), QU( 2278309315841980139ULL), + QU( 4329765449648304839ULL), QU( 5963108095785485298ULL), + QU( 4880024847478722478ULL), QU(16015608779890240947ULL), + QU( 1866679034261393544ULL), QU( 914821179919731519ULL), + QU( 9643404035648760131ULL), QU( 2418114953615593915ULL), + QU( 944756836073702374ULL), QU(15186388048737296834ULL), + QU( 7723355336128442206ULL), QU( 7500747479679599691ULL), + QU(18013961306453293634ULL), QU( 2315274808095756456ULL), + QU(13655308255424029566ULL), QU(17203800273561677098ULL), + QU( 1382158694422087756ULL), QU( 5090390250309588976ULL), + QU( 517170818384213989ULL), QU( 1612709252627729621ULL), + QU( 1330118955572449606ULL), QU( 300922478056709885ULL), + QU(18115693291289091987ULL), QU(13491407109725238321ULL), + QU(15293714633593827320ULL), QU( 5151539373053314504ULL), + QU( 5951523243743139207ULL), QU(14459112015249527975ULL), + QU( 5456113959000700739ULL), QU( 3877918438464873016ULL), + QU(12534071654260163555ULL), QU(15871678376893555041ULL), + QU(11005484805712025549ULL), QU(16353066973143374252ULL), + QU( 4358331472063256685ULL), QU( 8268349332210859288ULL), + QU(12485161590939658075ULL), QU(13955993592854471343ULL), + QU( 5911446886848367039ULL), QU(14925834086813706974ULL), + QU( 6590362597857994805ULL), QU( 1280544923533661875ULL), + QU( 1637756018947988164ULL), QU( 4734090064512686329ULL), + QU(16693705263131485912ULL), QU( 6834882340494360958ULL), + QU( 8120732176159658505ULL), QU( 2244371958905329346ULL), + QU(10447499707729734021ULL), QU( 7318742361446942194ULL), + QU( 8032857516355555296ULL), QU(14023605983059313116ULL), + QU( 1032336061815461376ULL), QU( 9840995337876562612ULL), + QU( 9869256223029203587ULL), QU(12227975697177267636ULL), + QU(12728115115844186033ULL), QU( 7752058479783205470ULL), + QU( 729733219713393087ULL), QU(12954017801239007622ULL) }; static const uint64_t init_by_array_64_expected[] = { - QU( 2100341266307895239LLU), QU( 8344256300489757943LLU), - QU(15687933285484243894LLU), QU( 8268620370277076319LLU), - QU(12371852309826545459LLU), QU( 8800491541730110238LLU), - QU(18113268950100835773LLU), QU( 2886823658884438119LLU), - QU( 3293667307248180724LLU), QU( 9307928143300172731LLU), - QU( 7688082017574293629LLU), QU( 900986224735166665LLU), - QU( 9977972710722265039LLU), QU( 6008205004994830552LLU), - QU( 546909104521689292LLU), QU( 7428471521869107594LLU), - QU(14777563419314721179LLU), QU(16116143076567350053LLU), - QU( 5322685342003142329LLU), QU( 4200427048445863473LLU), - QU( 4693092150132559146LLU), QU(13671425863759338582LLU), - QU( 6747117460737639916LLU), QU( 4732666080236551150LLU), - QU( 5912839950611941263LLU), QU( 3903717554504704909LLU), - QU( 2615667650256786818LLU), QU(10844129913887006352LLU), - QU(13786467861810997820LLU), QU(14267853002994021570LLU), - QU(13767807302847237439LLU), QU(16407963253707224617LLU), - QU( 4802498363698583497LLU), QU( 2523802839317209764LLU), - QU( 3822579397797475589LLU), QU( 8950320572212130610LLU), - QU( 3745623504978342534LLU), QU(16092609066068482806LLU), - QU( 9817016950274642398LLU), QU(10591660660323829098LLU), - QU(11751606650792815920LLU), QU( 5122873818577122211LLU), - QU(17209553764913936624LLU), QU( 6249057709284380343LLU), - QU(15088791264695071830LLU), QU(15344673071709851930LLU), - QU( 4345751415293646084LLU), QU( 2542865750703067928LLU), - QU(13520525127852368784LLU), QU(18294188662880997241LLU), - QU( 3871781938044881523LLU), QU( 2873487268122812184LLU), - QU(15099676759482679005LLU), QU(15442599127239350490LLU), - QU( 6311893274367710888LLU), QU( 3286118760484672933LLU), - QU( 4146067961333542189LLU), QU(13303942567897208770LLU), - QU( 8196013722255630418LLU), QU( 4437815439340979989LLU), - QU(15433791533450605135LLU), QU( 4254828956815687049LLU), - QU( 1310903207708286015LLU), QU(10529182764462398549LLU), - QU(14900231311660638810LLU), QU( 9727017277104609793LLU), - QU( 1821308310948199033LLU), QU(11628861435066772084LLU), - QU( 9469019138491546924LLU), QU( 3145812670532604988LLU), - QU( 9938468915045491919LLU), QU( 1562447430672662142LLU), - QU(13963995266697989134LLU), QU( 3356884357625028695LLU), - QU( 4499850304584309747LLU), QU( 8456825817023658122LLU), - QU(10859039922814285279LLU), QU( 8099512337972526555LLU), - QU( 348006375109672149LLU), QU(11919893998241688603LLU), - QU( 1104199577402948826LLU), QU(16689191854356060289LLU), - QU(10992552041730168078LLU), QU( 7243733172705465836LLU), - QU( 5668075606180319560LLU), QU(18182847037333286970LLU), - QU( 4290215357664631322LLU), QU( 4061414220791828613LLU), - QU(13006291061652989604LLU), QU( 7140491178917128798LLU), - QU(12703446217663283481LLU), QU( 5500220597564558267LLU), - QU(10330551509971296358LLU), QU(15958554768648714492LLU), - QU( 5174555954515360045LLU), QU( 1731318837687577735LLU), - QU( 3557700801048354857LLU), QU(13764012341928616198LLU), - QU(13115166194379119043LLU), QU( 7989321021560255519LLU), - QU( 2103584280905877040LLU), QU( 9230788662155228488LLU), - QU(16396629323325547654LLU), QU( 657926409811318051LLU), - QU(15046700264391400727LLU), QU( 5120132858771880830LLU), - QU( 7934160097989028561LLU), QU( 6963121488531976245LLU), - QU(17412329602621742089LLU), QU(15144843053931774092LLU), - QU(17204176651763054532LLU), QU(13166595387554065870LLU), - QU( 8590377810513960213LLU), QU( 5834365135373991938LLU), - QU( 7640913007182226243LLU), QU( 3479394703859418425LLU), - QU(16402784452644521040LLU), QU( 4993979809687083980LLU), - QU(13254522168097688865LLU), QU(15643659095244365219LLU), - QU( 5881437660538424982LLU), QU(11174892200618987379LLU), - QU( 254409966159711077LLU), QU(17158413043140549909LLU), - QU( 3638048789290376272LLU), QU( 1376816930299489190LLU), - QU( 4622462095217761923LLU), QU(15086407973010263515LLU), - QU(13253971772784692238LLU), QU( 5270549043541649236LLU), - QU(11182714186805411604LLU), QU(12283846437495577140LLU), - QU( 5297647149908953219LLU), QU(10047451738316836654LLU), - QU( 4938228100367874746LLU), QU(12328523025304077923LLU), - QU( 3601049438595312361LLU), QU( 9313624118352733770LLU), - QU(13322966086117661798LLU), QU(16660005705644029394LLU), - QU(11337677526988872373LLU), QU(13869299102574417795LLU), - QU(15642043183045645437LLU), QU( 3021755569085880019LLU), - QU( 4979741767761188161LLU), QU(13679979092079279587LLU), - QU( 3344685842861071743LLU), QU(13947960059899588104LLU), - QU( 305806934293368007LLU), QU( 5749173929201650029LLU), - QU(11123724852118844098LLU), QU(15128987688788879802LLU), - QU(15251651211024665009LLU), QU( 7689925933816577776LLU), - QU(16732804392695859449LLU), QU(17087345401014078468LLU), - QU(14315108589159048871LLU), QU( 4820700266619778917LLU), - QU(16709637539357958441LLU), QU( 4936227875177351374LLU), - QU( 2137907697912987247LLU), QU(11628565601408395420LLU), - QU( 2333250549241556786LLU), QU( 5711200379577778637LLU), - QU( 5170680131529031729LLU), QU(12620392043061335164LLU), - QU( 95363390101096078LLU), QU( 5487981914081709462LLU), - QU( 1763109823981838620LLU), QU( 3395861271473224396LLU), - QU( 1300496844282213595LLU), QU( 6894316212820232902LLU), - QU(10673859651135576674LLU), QU( 5911839658857903252LLU), - QU(17407110743387299102LLU), QU( 8257427154623140385LLU), - QU(11389003026741800267LLU), QU( 4070043211095013717LLU), - QU(11663806997145259025LLU), QU(15265598950648798210LLU), - QU( 630585789434030934LLU), QU( 3524446529213587334LLU), - QU( 7186424168495184211LLU), QU(10806585451386379021LLU), - QU(11120017753500499273LLU), QU( 1586837651387701301LLU), - QU(17530454400954415544LLU), QU( 9991670045077880430LLU), - QU( 7550997268990730180LLU), QU( 8640249196597379304LLU), - QU( 3522203892786893823LLU), QU(10401116549878854788LLU), - QU(13690285544733124852LLU), QU( 8295785675455774586LLU), - QU(15535716172155117603LLU), QU( 3112108583723722511LLU), - QU(17633179955339271113LLU), QU(18154208056063759375LLU), - QU( 1866409236285815666LLU), QU(13326075895396412882LLU), - QU( 8756261842948020025LLU), QU( 6281852999868439131LLU), - QU(15087653361275292858LLU), QU(10333923911152949397LLU), - QU( 5265567645757408500LLU), QU(12728041843210352184LLU), - QU( 6347959327507828759LLU), QU( 154112802625564758LLU), - QU(18235228308679780218LLU), QU( 3253805274673352418LLU), - QU( 4849171610689031197LLU), QU(17948529398340432518LLU), - QU(13803510475637409167LLU), QU(13506570190409883095LLU), - QU(15870801273282960805LLU), QU( 8451286481299170773LLU), - QU( 9562190620034457541LLU), QU( 8518905387449138364LLU), - QU(12681306401363385655LLU), QU( 3788073690559762558LLU), - QU( 5256820289573487769LLU), QU( 2752021372314875467LLU), - QU( 6354035166862520716LLU), QU( 4328956378309739069LLU), - QU( 449087441228269600LLU), QU( 5533508742653090868LLU), - QU( 1260389420404746988LLU), QU(18175394473289055097LLU), - QU( 1535467109660399420LLU), QU( 8818894282874061442LLU), - QU(12140873243824811213LLU), QU(15031386653823014946LLU), - QU( 1286028221456149232LLU), QU( 6329608889367858784LLU), - QU( 9419654354945132725LLU), QU( 6094576547061672379LLU), - QU(17706217251847450255LLU), QU( 1733495073065878126LLU), - QU(16918923754607552663LLU), QU( 8881949849954945044LLU), - QU(12938977706896313891LLU), QU(14043628638299793407LLU), - QU(18393874581723718233LLU), QU( 6886318534846892044LLU), - QU(14577870878038334081LLU), QU(13541558383439414119LLU), - QU(13570472158807588273LLU), QU(18300760537910283361LLU), - QU( 818368572800609205LLU), QU( 1417000585112573219LLU), - QU(12337533143867683655LLU), QU(12433180994702314480LLU), - QU( 778190005829189083LLU), QU(13667356216206524711LLU), - QU( 9866149895295225230LLU), QU(11043240490417111999LLU), - QU( 1123933826541378598LLU), QU( 6469631933605123610LLU), - QU(14508554074431980040LLU), QU(13918931242962026714LLU), - QU( 2870785929342348285LLU), QU(14786362626740736974LLU), - QU(13176680060902695786LLU), QU( 9591778613541679456LLU), - QU( 9097662885117436706LLU), QU( 749262234240924947LLU), - QU( 1944844067793307093LLU), QU( 4339214904577487742LLU), - QU( 8009584152961946551LLU), QU(16073159501225501777LLU), - QU( 3335870590499306217LLU), QU(17088312653151202847LLU), - QU( 3108893142681931848LLU), QU(16636841767202792021LLU), - QU(10423316431118400637LLU), QU( 8008357368674443506LLU), - QU(11340015231914677875LLU), QU(17687896501594936090LLU), - QU(15173627921763199958LLU), QU( 542569482243721959LLU), - QU(15071714982769812975LLU), QU( 4466624872151386956LLU), - QU( 1901780715602332461LLU), QU( 9822227742154351098LLU), - QU( 1479332892928648780LLU), QU( 6981611948382474400LLU), - QU( 7620824924456077376LLU), QU(14095973329429406782LLU), - QU( 7902744005696185404LLU), QU(15830577219375036920LLU), - QU(10287076667317764416LLU), QU(12334872764071724025LLU), - QU( 4419302088133544331LLU), QU(14455842851266090520LLU), - QU(12488077416504654222LLU), QU( 7953892017701886766LLU), - QU( 6331484925529519007LLU), QU( 4902145853785030022LLU), - QU(17010159216096443073LLU), QU(11945354668653886087LLU), - QU(15112022728645230829LLU), QU(17363484484522986742LLU), - QU( 4423497825896692887LLU), QU( 8155489510809067471LLU), - QU( 258966605622576285LLU), QU( 5462958075742020534LLU), - QU( 6763710214913276228LLU), QU( 2368935183451109054LLU), - QU(14209506165246453811LLU), QU( 2646257040978514881LLU), - QU( 3776001911922207672LLU), QU( 1419304601390147631LLU), - QU(14987366598022458284LLU), QU( 3977770701065815721LLU), - QU( 730820417451838898LLU), QU( 3982991703612885327LLU), - QU( 2803544519671388477LLU), QU(17067667221114424649LLU), - QU( 2922555119737867166LLU), QU( 1989477584121460932LLU), - QU(15020387605892337354LLU), QU( 9293277796427533547LLU), - QU(10722181424063557247LLU), QU(16704542332047511651LLU), - QU( 5008286236142089514LLU), QU(16174732308747382540LLU), - QU(17597019485798338402LLU), QU(13081745199110622093LLU), - QU( 8850305883842258115LLU), QU(12723629125624589005LLU), - QU( 8140566453402805978LLU), QU(15356684607680935061LLU), - QU(14222190387342648650LLU), QU(11134610460665975178LLU), - QU( 1259799058620984266LLU), QU(13281656268025610041LLU), - QU( 298262561068153992LLU), QU(12277871700239212922LLU), - QU(13911297774719779438LLU), QU(16556727962761474934LLU), - QU(17903010316654728010LLU), QU( 9682617699648434744LLU), - QU(14757681836838592850LLU), QU( 1327242446558524473LLU), - QU(11126645098780572792LLU), QU( 1883602329313221774LLU), - QU( 2543897783922776873LLU), QU(15029168513767772842LLU), - QU(12710270651039129878LLU), QU(16118202956069604504LLU), - QU(15010759372168680524LLU), QU( 2296827082251923948LLU), - QU(10793729742623518101LLU), QU(13829764151845413046LLU), - QU(17769301223184451213LLU), QU( 3118268169210783372LLU), - QU(17626204544105123127LLU), QU( 7416718488974352644LLU), - QU(10450751996212925994LLU), QU( 9352529519128770586LLU), - QU( 259347569641110140LLU), QU( 8048588892269692697LLU), - QU( 1774414152306494058LLU), QU(10669548347214355622LLU), - QU(13061992253816795081LLU), QU(18432677803063861659LLU), - QU( 8879191055593984333LLU), QU(12433753195199268041LLU), - QU(14919392415439730602LLU), QU( 6612848378595332963LLU), - QU( 6320986812036143628LLU), QU(10465592420226092859LLU), - QU( 4196009278962570808LLU), QU( 3747816564473572224LLU), - QU(17941203486133732898LLU), QU( 2350310037040505198LLU), - QU( 5811779859134370113LLU), QU(10492109599506195126LLU), - QU( 7699650690179541274LLU), QU( 1954338494306022961LLU), - QU(14095816969027231152LLU), QU( 5841346919964852061LLU), - QU(14945969510148214735LLU), QU( 3680200305887550992LLU), - QU( 6218047466131695792LLU), QU( 8242165745175775096LLU), - QU(11021371934053307357LLU), QU( 1265099502753169797LLU), - QU( 4644347436111321718LLU), QU( 3609296916782832859LLU), - QU( 8109807992218521571LLU), QU(18387884215648662020LLU), - QU(14656324896296392902LLU), QU(17386819091238216751LLU), - QU(17788300878582317152LLU), QU( 7919446259742399591LLU), - QU( 4466613134576358004LLU), QU(12928181023667938509LLU), - QU(13147446154454932030LLU), QU(16552129038252734620LLU), - QU( 8395299403738822450LLU), QU(11313817655275361164LLU), - QU( 434258809499511718LLU), QU( 2074882104954788676LLU), - QU( 7929892178759395518LLU), QU( 9006461629105745388LLU), - QU( 5176475650000323086LLU), QU(11128357033468341069LLU), - QU(12026158851559118955LLU), QU(14699716249471156500LLU), - QU( 448982497120206757LLU), QU( 4156475356685519900LLU), - QU( 6063816103417215727LLU), QU(10073289387954971479LLU), - QU( 8174466846138590962LLU), QU( 2675777452363449006LLU), - QU( 9090685420572474281LLU), QU( 6659652652765562060LLU), - QU(12923120304018106621LLU), QU(11117480560334526775LLU), - QU( 937910473424587511LLU), QU( 1838692113502346645LLU), - QU(11133914074648726180LLU), QU( 7922600945143884053LLU), - QU(13435287702700959550LLU), QU( 5287964921251123332LLU), - QU(11354875374575318947LLU), QU(17955724760748238133LLU), - QU(13728617396297106512LLU), QU( 4107449660118101255LLU), - QU( 1210269794886589623LLU), QU(11408687205733456282LLU), - QU( 4538354710392677887LLU), QU(13566803319341319267LLU), - QU(17870798107734050771LLU), QU( 3354318982568089135LLU), - QU( 9034450839405133651LLU), QU(13087431795753424314LLU), - QU( 950333102820688239LLU), QU( 1968360654535604116LLU), - QU(16840551645563314995LLU), QU( 8867501803892924995LLU), - QU(11395388644490626845LLU), QU( 1529815836300732204LLU), - QU(13330848522996608842LLU), QU( 1813432878817504265LLU), - QU( 2336867432693429560LLU), QU(15192805445973385902LLU), - QU( 2528593071076407877LLU), QU( 128459777936689248LLU), - QU( 9976345382867214866LLU), QU( 6208885766767996043LLU), - QU(14982349522273141706LLU), QU( 3099654362410737822LLU), - QU(13776700761947297661LLU), QU( 8806185470684925550LLU), - QU( 8151717890410585321LLU), QU( 640860591588072925LLU), - QU(14592096303937307465LLU), QU( 9056472419613564846LLU), - QU(14861544647742266352LLU), QU(12703771500398470216LLU), - QU( 3142372800384138465LLU), QU( 6201105606917248196LLU), - QU(18337516409359270184LLU), QU(15042268695665115339LLU), - QU(15188246541383283846LLU), QU(12800028693090114519LLU), - QU( 5992859621101493472LLU), QU(18278043971816803521LLU), - QU( 9002773075219424560LLU), QU( 7325707116943598353LLU), - QU( 7930571931248040822LLU), QU( 5645275869617023448LLU), - QU( 7266107455295958487LLU), QU( 4363664528273524411LLU), - QU(14313875763787479809LLU), QU(17059695613553486802LLU), - QU( 9247761425889940932LLU), QU(13704726459237593128LLU), - QU( 2701312427328909832LLU), QU(17235532008287243115LLU), - QU(14093147761491729538LLU), QU( 6247352273768386516LLU), - QU( 8268710048153268415LLU), QU( 7985295214477182083LLU), - QU(15624495190888896807LLU), QU( 3772753430045262788LLU), - QU( 9133991620474991698LLU), QU( 5665791943316256028LLU), - QU( 7551996832462193473LLU), QU(13163729206798953877LLU), - QU( 9263532074153846374LLU), QU( 1015460703698618353LLU), - QU(17929874696989519390LLU), QU(18257884721466153847LLU), - QU(16271867543011222991LLU), QU( 3905971519021791941LLU), - QU(16814488397137052085LLU), QU( 1321197685504621613LLU), - QU( 2870359191894002181LLU), QU(14317282970323395450LLU), - QU(13663920845511074366LLU), QU( 2052463995796539594LLU), - QU(14126345686431444337LLU), QU( 1727572121947022534LLU), - QU(17793552254485594241LLU), QU( 6738857418849205750LLU), - QU( 1282987123157442952LLU), QU(16655480021581159251LLU), - QU( 6784587032080183866LLU), QU(14726758805359965162LLU), - QU( 7577995933961987349LLU), QU(12539609320311114036LLU), - QU(10789773033385439494LLU), QU( 8517001497411158227LLU), - QU(10075543932136339710LLU), QU(14838152340938811081LLU), - QU( 9560840631794044194LLU), QU(17445736541454117475LLU), - QU(10633026464336393186LLU), QU(15705729708242246293LLU), - QU( 1117517596891411098LLU), QU( 4305657943415886942LLU), - QU( 4948856840533979263LLU), QU(16071681989041789593LLU), - QU(13723031429272486527LLU), QU( 7639567622306509462LLU), - QU(12670424537483090390LLU), QU( 9715223453097197134LLU), - QU( 5457173389992686394LLU), QU( 289857129276135145LLU), - QU(17048610270521972512LLU), QU( 692768013309835485LLU), - QU(14823232360546632057LLU), QU(18218002361317895936LLU), - QU( 3281724260212650204LLU), QU(16453957266549513795LLU), - QU( 8592711109774511881LLU), QU( 929825123473369579LLU), - QU(15966784769764367791LLU), QU( 9627344291450607588LLU), - QU(10849555504977813287LLU), QU( 9234566913936339275LLU), - QU( 6413807690366911210LLU), QU(10862389016184219267LLU), - QU(13842504799335374048LLU), QU( 1531994113376881174LLU), - QU( 2081314867544364459LLU), QU(16430628791616959932LLU), - QU( 8314714038654394368LLU), QU( 9155473892098431813LLU), - QU(12577843786670475704LLU), QU( 4399161106452401017LLU), - QU( 1668083091682623186LLU), QU( 1741383777203714216LLU), - QU( 2162597285417794374LLU), QU(15841980159165218736LLU), - QU( 1971354603551467079LLU), QU( 1206714764913205968LLU), - QU( 4790860439591272330LLU), QU(14699375615594055799LLU), - QU( 8374423871657449988LLU), QU(10950685736472937738LLU), - QU( 697344331343267176LLU), QU(10084998763118059810LLU), - QU(12897369539795983124LLU), QU(12351260292144383605LLU), - QU( 1268810970176811234LLU), QU( 7406287800414582768LLU), - QU( 516169557043807831LLU), QU( 5077568278710520380LLU), - QU( 3828791738309039304LLU), QU( 7721974069946943610LLU), - QU( 3534670260981096460LLU), QU( 4865792189600584891LLU), - QU(16892578493734337298LLU), QU( 9161499464278042590LLU), - QU(11976149624067055931LLU), QU(13219479887277343990LLU), - QU(14161556738111500680LLU), QU(14670715255011223056LLU), - QU( 4671205678403576558LLU), QU(12633022931454259781LLU), - QU(14821376219869187646LLU), QU( 751181776484317028LLU), - QU( 2192211308839047070LLU), QU(11787306362361245189LLU), - QU(10672375120744095707LLU), QU( 4601972328345244467LLU), - QU(15457217788831125879LLU), QU( 8464345256775460809LLU), - QU(10191938789487159478LLU), QU( 6184348739615197613LLU), - QU(11425436778806882100LLU), QU( 2739227089124319793LLU), - QU( 461464518456000551LLU), QU( 4689850170029177442LLU), - QU( 6120307814374078625LLU), QU(11153579230681708671LLU), - QU( 7891721473905347926LLU), QU(10281646937824872400LLU), - QU( 3026099648191332248LLU), QU( 8666750296953273818LLU), - QU(14978499698844363232LLU), QU(13303395102890132065LLU), - QU( 8182358205292864080LLU), QU(10560547713972971291LLU), - QU(11981635489418959093LLU), QU( 3134621354935288409LLU), - QU(11580681977404383968LLU), QU(14205530317404088650LLU), - QU( 5997789011854923157LLU), QU(13659151593432238041LLU), - QU(11664332114338865086LLU), QU( 7490351383220929386LLU), - QU( 7189290499881530378LLU), QU(15039262734271020220LLU), - QU( 2057217285976980055LLU), QU( 555570804905355739LLU), - QU(11235311968348555110LLU), QU(13824557146269603217LLU), - QU(16906788840653099693LLU), QU( 7222878245455661677LLU), - QU( 5245139444332423756LLU), QU( 4723748462805674292LLU), - QU(12216509815698568612LLU), QU(17402362976648951187LLU), - QU(17389614836810366768LLU), QU( 4880936484146667711LLU), - QU( 9085007839292639880LLU), QU(13837353458498535449LLU), - QU(11914419854360366677LLU), QU(16595890135313864103LLU), - QU( 6313969847197627222LLU), QU(18296909792163910431LLU), - QU(10041780113382084042LLU), QU( 2499478551172884794LLU), - QU(11057894246241189489LLU), QU( 9742243032389068555LLU), - QU(12838934582673196228LLU), QU(13437023235248490367LLU), - QU(13372420669446163240LLU), QU( 6752564244716909224LLU), - QU( 7157333073400313737LLU), QU(12230281516370654308LLU), - QU( 1182884552219419117LLU), QU( 2955125381312499218LLU), - QU(10308827097079443249LLU), QU( 1337648572986534958LLU), - QU(16378788590020343939LLU), QU( 108619126514420935LLU), - QU( 3990981009621629188LLU), QU( 5460953070230946410LLU), - QU( 9703328329366531883LLU), QU(13166631489188077236LLU), - QU( 1104768831213675170LLU), QU( 3447930458553877908LLU), - QU( 8067172487769945676LLU), QU( 5445802098190775347LLU), - QU( 3244840981648973873LLU), QU(17314668322981950060LLU), - QU( 5006812527827763807LLU), QU(18158695070225526260LLU), - QU( 2824536478852417853LLU), QU(13974775809127519886LLU), - QU( 9814362769074067392LLU), QU(17276205156374862128LLU), - QU(11361680725379306967LLU), QU( 3422581970382012542LLU), - QU(11003189603753241266LLU), QU(11194292945277862261LLU), - QU( 6839623313908521348LLU), QU(11935326462707324634LLU), - QU( 1611456788685878444LLU), QU(13112620989475558907LLU), - QU( 517659108904450427LLU), QU(13558114318574407624LLU), - QU(15699089742731633077LLU), QU( 4988979278862685458LLU), - QU( 8111373583056521297LLU), QU( 3891258746615399627LLU), - QU( 8137298251469718086LLU), QU(12748663295624701649LLU), - QU( 4389835683495292062LLU), QU( 5775217872128831729LLU), - QU( 9462091896405534927LLU), QU( 8498124108820263989LLU), - QU( 8059131278842839525LLU), QU(10503167994254090892LLU), - QU(11613153541070396656LLU), QU(18069248738504647790LLU), - QU( 570657419109768508LLU), QU( 3950574167771159665LLU), - QU( 5514655599604313077LLU), QU( 2908460854428484165LLU), - QU(10777722615935663114LLU), QU(12007363304839279486LLU), - QU( 9800646187569484767LLU), QU( 8795423564889864287LLU), - QU(14257396680131028419LLU), QU( 6405465117315096498LLU), - QU( 7939411072208774878LLU), QU(17577572378528990006LLU), - QU(14785873806715994850LLU), QU(16770572680854747390LLU), - QU(18127549474419396481LLU), QU(11637013449455757750LLU), - QU(14371851933996761086LLU), QU( 3601181063650110280LLU), - QU( 4126442845019316144LLU), QU(10198287239244320669LLU), - QU(18000169628555379659LLU), QU(18392482400739978269LLU), - QU( 6219919037686919957LLU), QU( 3610085377719446052LLU), - QU( 2513925039981776336LLU), QU(16679413537926716955LLU), - QU(12903302131714909434LLU), QU( 5581145789762985009LLU), - QU(12325955044293303233LLU), QU(17216111180742141204LLU), - QU( 6321919595276545740LLU), QU( 3507521147216174501LLU), - QU( 9659194593319481840LLU), QU(11473976005975358326LLU), - QU(14742730101435987026LLU), QU( 492845897709954780LLU), - QU(16976371186162599676LLU), QU(17712703422837648655LLU), - QU( 9881254778587061697LLU), QU( 8413223156302299551LLU), - QU( 1563841828254089168LLU), QU( 9996032758786671975LLU), - QU( 138877700583772667LLU), QU(13003043368574995989LLU), - QU( 4390573668650456587LLU), QU( 8610287390568126755LLU), - QU(15126904974266642199LLU), QU( 6703637238986057662LLU), - QU( 2873075592956810157LLU), QU( 6035080933946049418LLU), - QU(13382846581202353014LLU), QU( 7303971031814642463LLU), - QU(18418024405307444267LLU), QU( 5847096731675404647LLU), - QU( 4035880699639842500LLU), QU(11525348625112218478LLU), - QU( 3041162365459574102LLU), QU( 2604734487727986558LLU), - QU(15526341771636983145LLU), QU(14556052310697370254LLU), - QU(12997787077930808155LLU), QU( 9601806501755554499LLU), - QU(11349677952521423389LLU), QU(14956777807644899350LLU), - QU(16559736957742852721LLU), QU(12360828274778140726LLU), - QU( 6685373272009662513LLU), QU(16932258748055324130LLU), - QU(15918051131954158508LLU), QU( 1692312913140790144LLU), - QU( 546653826801637367LLU), QU( 5341587076045986652LLU), - QU(14975057236342585662LLU), QU(12374976357340622412LLU), - QU(10328833995181940552LLU), QU(12831807101710443149LLU), - QU(10548514914382545716LLU), QU( 2217806727199715993LLU), - QU(12627067369242845138LLU), QU( 4598965364035438158LLU), - QU( 150923352751318171LLU), QU(14274109544442257283LLU), - QU( 4696661475093863031LLU), QU( 1505764114384654516LLU), - QU(10699185831891495147LLU), QU( 2392353847713620519LLU), - QU( 3652870166711788383LLU), QU( 8640653276221911108LLU), - QU( 3894077592275889704LLU), QU( 4918592872135964845LLU), - QU(16379121273281400789LLU), QU(12058465483591683656LLU), - QU(11250106829302924945LLU), QU( 1147537556296983005LLU), - QU( 6376342756004613268LLU), QU(14967128191709280506LLU), - QU(18007449949790627628LLU), QU( 9497178279316537841LLU), - QU( 7920174844809394893LLU), QU(10037752595255719907LLU), - QU(15875342784985217697LLU), QU(15311615921712850696LLU), - QU( 9552902652110992950LLU), QU(14054979450099721140LLU), - QU( 5998709773566417349LLU), QU(18027910339276320187LLU), - QU( 8223099053868585554LLU), QU( 7842270354824999767LLU), - QU( 4896315688770080292LLU), QU(12969320296569787895LLU), - QU( 2674321489185759961LLU), QU( 4053615936864718439LLU), - QU(11349775270588617578LLU), QU( 4743019256284553975LLU), - QU( 5602100217469723769LLU), QU(14398995691411527813LLU), - QU( 7412170493796825470LLU), QU( 836262406131744846LLU), - QU( 8231086633845153022LLU), QU( 5161377920438552287LLU), - QU( 8828731196169924949LLU), QU(16211142246465502680LLU), - QU( 3307990879253687818LLU), QU( 5193405406899782022LLU), - QU( 8510842117467566693LLU), QU( 6070955181022405365LLU), - QU(14482950231361409799LLU), QU(12585159371331138077LLU), - QU( 3511537678933588148LLU), QU( 2041849474531116417LLU), - QU(10944936685095345792LLU), QU(18303116923079107729LLU), - QU( 2720566371239725320LLU), QU( 4958672473562397622LLU), - QU( 3032326668253243412LLU), QU(13689418691726908338LLU), - QU( 1895205511728843996LLU), QU( 8146303515271990527LLU), - QU(16507343500056113480LLU), QU( 473996939105902919LLU), - QU( 9897686885246881481LLU), QU(14606433762712790575LLU), - QU( 6732796251605566368LLU), QU( 1399778120855368916LLU), - QU( 935023885182833777LLU), QU(16066282816186753477LLU), - QU( 7291270991820612055LLU), QU(17530230393129853844LLU), - QU(10223493623477451366LLU), QU(15841725630495676683LLU), - QU(17379567246435515824LLU), QU( 8588251429375561971LLU), - QU(18339511210887206423LLU), QU(17349587430725976100LLU), - QU(12244876521394838088LLU), QU( 6382187714147161259LLU), - QU(12335807181848950831LLU), QU(16948885622305460665LLU), - QU(13755097796371520506LLU), QU(14806740373324947801LLU), - QU( 4828699633859287703LLU), QU( 8209879281452301604LLU), - QU(12435716669553736437LLU), QU(13970976859588452131LLU), - QU( 6233960842566773148LLU), QU(12507096267900505759LLU), - QU( 1198713114381279421LLU), QU(14989862731124149015LLU), - QU(15932189508707978949LLU), QU( 2526406641432708722LLU), - QU( 29187427817271982LLU), QU( 1499802773054556353LLU), - QU(10816638187021897173LLU), QU( 5436139270839738132LLU), - QU( 6659882287036010082LLU), QU( 2154048955317173697LLU), - QU(10887317019333757642LLU), QU(16281091802634424955LLU), - QU(10754549879915384901LLU), QU(10760611745769249815LLU), - QU( 2161505946972504002LLU), QU( 5243132808986265107LLU), - QU(10129852179873415416LLU), QU( 710339480008649081LLU), - QU( 7802129453068808528LLU), QU(17967213567178907213LLU), - QU(15730859124668605599LLU), QU(13058356168962376502LLU), - QU( 3701224985413645909LLU), QU(14464065869149109264LLU), - QU( 9959272418844311646LLU), QU(10157426099515958752LLU), - QU(14013736814538268528LLU), QU(17797456992065653951LLU), - QU(17418878140257344806LLU), QU(15457429073540561521LLU), - QU( 2184426881360949378LLU), QU( 2062193041154712416LLU), - QU( 8553463347406931661LLU), QU( 4913057625202871854LLU), - QU( 2668943682126618425LLU), QU(17064444737891172288LLU), - QU( 4997115903913298637LLU), QU(12019402608892327416LLU), - QU(17603584559765897352LLU), QU(11367529582073647975LLU), - QU( 8211476043518436050LLU), QU( 8676849804070323674LLU), - QU(18431829230394475730LLU), QU(10490177861361247904LLU), - QU( 9508720602025651349LLU), QU( 7409627448555722700LLU), - QU( 5804047018862729008LLU), QU(11943858176893142594LLU), - QU(11908095418933847092LLU), QU( 5415449345715887652LLU), - QU( 1554022699166156407LLU), QU( 9073322106406017161LLU), - QU( 7080630967969047082LLU), QU(18049736940860732943LLU), - QU(12748714242594196794LLU), QU( 1226992415735156741LLU), - QU(17900981019609531193LLU), QU(11720739744008710999LLU), - QU( 3006400683394775434LLU), QU(11347974011751996028LLU), - QU( 3316999628257954608LLU), QU( 8384484563557639101LLU), - QU(18117794685961729767LLU), QU( 1900145025596618194LLU), - QU(17459527840632892676LLU), QU( 5634784101865710994LLU), - QU( 7918619300292897158LLU), QU( 3146577625026301350LLU), - QU( 9955212856499068767LLU), QU( 1873995843681746975LLU), - QU( 1561487759967972194LLU), QU( 8322718804375878474LLU), - QU(11300284215327028366LLU), QU( 4667391032508998982LLU), - QU( 9820104494306625580LLU), QU(17922397968599970610LLU), - QU( 1784690461886786712LLU), QU(14940365084341346821LLU), - QU( 5348719575594186181LLU), QU(10720419084507855261LLU), - QU(14210394354145143274LLU), QU( 2426468692164000131LLU), - QU(16271062114607059202LLU), QU(14851904092357070247LLU), - QU( 6524493015693121897LLU), QU( 9825473835127138531LLU), - QU(14222500616268569578LLU), QU(15521484052007487468LLU), - QU(14462579404124614699LLU), QU(11012375590820665520LLU), - QU(11625327350536084927LLU), QU(14452017765243785417LLU), - QU( 9989342263518766305LLU), QU( 3640105471101803790LLU), - QU( 4749866455897513242LLU), QU(13963064946736312044LLU), - QU(10007416591973223791LLU), QU(18314132234717431115LLU), - QU( 3286596588617483450LLU), QU( 7726163455370818765LLU), - QU( 7575454721115379328LLU), QU( 5308331576437663422LLU), - QU(18288821894903530934LLU), QU( 8028405805410554106LLU), - QU(15744019832103296628LLU), QU( 149765559630932100LLU), - QU( 6137705557200071977LLU), QU(14513416315434803615LLU), - QU(11665702820128984473LLU), QU( 218926670505601386LLU), - QU( 6868675028717769519LLU), QU(15282016569441512302LLU), - QU( 5707000497782960236LLU), QU( 6671120586555079567LLU), - QU( 2194098052618985448LLU), QU(16849577895477330978LLU), - QU(12957148471017466283LLU), QU( 1997805535404859393LLU), - QU( 1180721060263860490LLU), QU(13206391310193756958LLU), - QU(12980208674461861797LLU), QU( 3825967775058875366LLU), - QU(17543433670782042631LLU), QU( 1518339070120322730LLU), - QU(16344584340890991669LLU), QU( 2611327165318529819LLU), - QU(11265022723283422529LLU), QU( 4001552800373196817LLU), - QU(14509595890079346161LLU), QU( 3528717165416234562LLU), - QU(18153222571501914072LLU), QU( 9387182977209744425LLU), - QU(10064342315985580021LLU), QU(11373678413215253977LLU), - QU( 2308457853228798099LLU), QU( 9729042942839545302LLU), - QU( 7833785471140127746LLU), QU( 6351049900319844436LLU), - QU(14454610627133496067LLU), QU(12533175683634819111LLU), - QU(15570163926716513029LLU), QU(13356980519185762498LLU) + QU( 2100341266307895239ULL), QU( 8344256300489757943ULL), + QU(15687933285484243894ULL), QU( 8268620370277076319ULL), + QU(12371852309826545459ULL), QU( 8800491541730110238ULL), + QU(18113268950100835773ULL), QU( 2886823658884438119ULL), + QU( 3293667307248180724ULL), QU( 9307928143300172731ULL), + QU( 7688082017574293629ULL), QU( 900986224735166665ULL), + QU( 9977972710722265039ULL), QU( 6008205004994830552ULL), + QU( 546909104521689292ULL), QU( 7428471521869107594ULL), + QU(14777563419314721179ULL), QU(16116143076567350053ULL), + QU( 5322685342003142329ULL), QU( 4200427048445863473ULL), + QU( 4693092150132559146ULL), QU(13671425863759338582ULL), + QU( 6747117460737639916ULL), QU( 4732666080236551150ULL), + QU( 5912839950611941263ULL), QU( 3903717554504704909ULL), + QU( 2615667650256786818ULL), QU(10844129913887006352ULL), + QU(13786467861810997820ULL), QU(14267853002994021570ULL), + QU(13767807302847237439ULL), QU(16407963253707224617ULL), + QU( 4802498363698583497ULL), QU( 2523802839317209764ULL), + QU( 3822579397797475589ULL), QU( 8950320572212130610ULL), + QU( 3745623504978342534ULL), QU(16092609066068482806ULL), + QU( 9817016950274642398ULL), QU(10591660660323829098ULL), + QU(11751606650792815920ULL), QU( 5122873818577122211ULL), + QU(17209553764913936624ULL), QU( 6249057709284380343ULL), + QU(15088791264695071830ULL), QU(15344673071709851930ULL), + QU( 4345751415293646084ULL), QU( 2542865750703067928ULL), + QU(13520525127852368784ULL), QU(18294188662880997241ULL), + QU( 3871781938044881523ULL), QU( 2873487268122812184ULL), + QU(15099676759482679005ULL), QU(15442599127239350490ULL), + QU( 6311893274367710888ULL), QU( 3286118760484672933ULL), + QU( 4146067961333542189ULL), QU(13303942567897208770ULL), + QU( 8196013722255630418ULL), QU( 4437815439340979989ULL), + QU(15433791533450605135ULL), QU( 4254828956815687049ULL), + QU( 1310903207708286015ULL), QU(10529182764462398549ULL), + QU(14900231311660638810ULL), QU( 9727017277104609793ULL), + QU( 1821308310948199033ULL), QU(11628861435066772084ULL), + QU( 9469019138491546924ULL), QU( 3145812670532604988ULL), + QU( 9938468915045491919ULL), QU( 1562447430672662142ULL), + QU(13963995266697989134ULL), QU( 3356884357625028695ULL), + QU( 4499850304584309747ULL), QU( 8456825817023658122ULL), + QU(10859039922814285279ULL), QU( 8099512337972526555ULL), + QU( 348006375109672149ULL), QU(11919893998241688603ULL), + QU( 1104199577402948826ULL), QU(16689191854356060289ULL), + QU(10992552041730168078ULL), QU( 7243733172705465836ULL), + QU( 5668075606180319560ULL), QU(18182847037333286970ULL), + QU( 4290215357664631322ULL), QU( 4061414220791828613ULL), + QU(13006291061652989604ULL), QU( 7140491178917128798ULL), + QU(12703446217663283481ULL), QU( 5500220597564558267ULL), + QU(10330551509971296358ULL), QU(15958554768648714492ULL), + QU( 5174555954515360045ULL), QU( 1731318837687577735ULL), + QU( 3557700801048354857ULL), QU(13764012341928616198ULL), + QU(13115166194379119043ULL), QU( 7989321021560255519ULL), + QU( 2103584280905877040ULL), QU( 9230788662155228488ULL), + QU(16396629323325547654ULL), QU( 657926409811318051ULL), + QU(15046700264391400727ULL), QU( 5120132858771880830ULL), + QU( 7934160097989028561ULL), QU( 6963121488531976245ULL), + QU(17412329602621742089ULL), QU(15144843053931774092ULL), + QU(17204176651763054532ULL), QU(13166595387554065870ULL), + QU( 8590377810513960213ULL), QU( 5834365135373991938ULL), + QU( 7640913007182226243ULL), QU( 3479394703859418425ULL), + QU(16402784452644521040ULL), QU( 4993979809687083980ULL), + QU(13254522168097688865ULL), QU(15643659095244365219ULL), + QU( 5881437660538424982ULL), QU(11174892200618987379ULL), + QU( 254409966159711077ULL), QU(17158413043140549909ULL), + QU( 3638048789290376272ULL), QU( 1376816930299489190ULL), + QU( 4622462095217761923ULL), QU(15086407973010263515ULL), + QU(13253971772784692238ULL), QU( 5270549043541649236ULL), + QU(11182714186805411604ULL), QU(12283846437495577140ULL), + QU( 5297647149908953219ULL), QU(10047451738316836654ULL), + QU( 4938228100367874746ULL), QU(12328523025304077923ULL), + QU( 3601049438595312361ULL), QU( 9313624118352733770ULL), + QU(13322966086117661798ULL), QU(16660005705644029394ULL), + QU(11337677526988872373ULL), QU(13869299102574417795ULL), + QU(15642043183045645437ULL), QU( 3021755569085880019ULL), + QU( 4979741767761188161ULL), QU(13679979092079279587ULL), + QU( 3344685842861071743ULL), QU(13947960059899588104ULL), + QU( 305806934293368007ULL), QU( 5749173929201650029ULL), + QU(11123724852118844098ULL), QU(15128987688788879802ULL), + QU(15251651211024665009ULL), QU( 7689925933816577776ULL), + QU(16732804392695859449ULL), QU(17087345401014078468ULL), + QU(14315108589159048871ULL), QU( 4820700266619778917ULL), + QU(16709637539357958441ULL), QU( 4936227875177351374ULL), + QU( 2137907697912987247ULL), QU(11628565601408395420ULL), + QU( 2333250549241556786ULL), QU( 5711200379577778637ULL), + QU( 5170680131529031729ULL), QU(12620392043061335164ULL), + QU( 95363390101096078ULL), QU( 5487981914081709462ULL), + QU( 1763109823981838620ULL), QU( 3395861271473224396ULL), + QU( 1300496844282213595ULL), QU( 6894316212820232902ULL), + QU(10673859651135576674ULL), QU( 5911839658857903252ULL), + QU(17407110743387299102ULL), QU( 8257427154623140385ULL), + QU(11389003026741800267ULL), QU( 4070043211095013717ULL), + QU(11663806997145259025ULL), QU(15265598950648798210ULL), + QU( 630585789434030934ULL), QU( 3524446529213587334ULL), + QU( 7186424168495184211ULL), QU(10806585451386379021ULL), + QU(11120017753500499273ULL), QU( 1586837651387701301ULL), + QU(17530454400954415544ULL), QU( 9991670045077880430ULL), + QU( 7550997268990730180ULL), QU( 8640249196597379304ULL), + QU( 3522203892786893823ULL), QU(10401116549878854788ULL), + QU(13690285544733124852ULL), QU( 8295785675455774586ULL), + QU(15535716172155117603ULL), QU( 3112108583723722511ULL), + QU(17633179955339271113ULL), QU(18154208056063759375ULL), + QU( 1866409236285815666ULL), QU(13326075895396412882ULL), + QU( 8756261842948020025ULL), QU( 6281852999868439131ULL), + QU(15087653361275292858ULL), QU(10333923911152949397ULL), + QU( 5265567645757408500ULL), QU(12728041843210352184ULL), + QU( 6347959327507828759ULL), QU( 154112802625564758ULL), + QU(18235228308679780218ULL), QU( 3253805274673352418ULL), + QU( 4849171610689031197ULL), QU(17948529398340432518ULL), + QU(13803510475637409167ULL), QU(13506570190409883095ULL), + QU(15870801273282960805ULL), QU( 8451286481299170773ULL), + QU( 9562190620034457541ULL), QU( 8518905387449138364ULL), + QU(12681306401363385655ULL), QU( 3788073690559762558ULL), + QU( 5256820289573487769ULL), QU( 2752021372314875467ULL), + QU( 6354035166862520716ULL), QU( 4328956378309739069ULL), + QU( 449087441228269600ULL), QU( 5533508742653090868ULL), + QU( 1260389420404746988ULL), QU(18175394473289055097ULL), + QU( 1535467109660399420ULL), QU( 8818894282874061442ULL), + QU(12140873243824811213ULL), QU(15031386653823014946ULL), + QU( 1286028221456149232ULL), QU( 6329608889367858784ULL), + QU( 9419654354945132725ULL), QU( 6094576547061672379ULL), + QU(17706217251847450255ULL), QU( 1733495073065878126ULL), + QU(16918923754607552663ULL), QU( 8881949849954945044ULL), + QU(12938977706896313891ULL), QU(14043628638299793407ULL), + QU(18393874581723718233ULL), QU( 6886318534846892044ULL), + QU(14577870878038334081ULL), QU(13541558383439414119ULL), + QU(13570472158807588273ULL), QU(18300760537910283361ULL), + QU( 818368572800609205ULL), QU( 1417000585112573219ULL), + QU(12337533143867683655ULL), QU(12433180994702314480ULL), + QU( 778190005829189083ULL), QU(13667356216206524711ULL), + QU( 9866149895295225230ULL), QU(11043240490417111999ULL), + QU( 1123933826541378598ULL), QU( 6469631933605123610ULL), + QU(14508554074431980040ULL), QU(13918931242962026714ULL), + QU( 2870785929342348285ULL), QU(14786362626740736974ULL), + QU(13176680060902695786ULL), QU( 9591778613541679456ULL), + QU( 9097662885117436706ULL), QU( 749262234240924947ULL), + QU( 1944844067793307093ULL), QU( 4339214904577487742ULL), + QU( 8009584152961946551ULL), QU(16073159501225501777ULL), + QU( 3335870590499306217ULL), QU(17088312653151202847ULL), + QU( 3108893142681931848ULL), QU(16636841767202792021ULL), + QU(10423316431118400637ULL), QU( 8008357368674443506ULL), + QU(11340015231914677875ULL), QU(17687896501594936090ULL), + QU(15173627921763199958ULL), QU( 542569482243721959ULL), + QU(15071714982769812975ULL), QU( 4466624872151386956ULL), + QU( 1901780715602332461ULL), QU( 9822227742154351098ULL), + QU( 1479332892928648780ULL), QU( 6981611948382474400ULL), + QU( 7620824924456077376ULL), QU(14095973329429406782ULL), + QU( 7902744005696185404ULL), QU(15830577219375036920ULL), + QU(10287076667317764416ULL), QU(12334872764071724025ULL), + QU( 4419302088133544331ULL), QU(14455842851266090520ULL), + QU(12488077416504654222ULL), QU( 7953892017701886766ULL), + QU( 6331484925529519007ULL), QU( 4902145853785030022ULL), + QU(17010159216096443073ULL), QU(11945354668653886087ULL), + QU(15112022728645230829ULL), QU(17363484484522986742ULL), + QU( 4423497825896692887ULL), QU( 8155489510809067471ULL), + QU( 258966605622576285ULL), QU( 5462958075742020534ULL), + QU( 6763710214913276228ULL), QU( 2368935183451109054ULL), + QU(14209506165246453811ULL), QU( 2646257040978514881ULL), + QU( 3776001911922207672ULL), QU( 1419304601390147631ULL), + QU(14987366598022458284ULL), QU( 3977770701065815721ULL), + QU( 730820417451838898ULL), QU( 3982991703612885327ULL), + QU( 2803544519671388477ULL), QU(17067667221114424649ULL), + QU( 2922555119737867166ULL), QU( 1989477584121460932ULL), + QU(15020387605892337354ULL), QU( 9293277796427533547ULL), + QU(10722181424063557247ULL), QU(16704542332047511651ULL), + QU( 5008286236142089514ULL), QU(16174732308747382540ULL), + QU(17597019485798338402ULL), QU(13081745199110622093ULL), + QU( 8850305883842258115ULL), QU(12723629125624589005ULL), + QU( 8140566453402805978ULL), QU(15356684607680935061ULL), + QU(14222190387342648650ULL), QU(11134610460665975178ULL), + QU( 1259799058620984266ULL), QU(13281656268025610041ULL), + QU( 298262561068153992ULL), QU(12277871700239212922ULL), + QU(13911297774719779438ULL), QU(16556727962761474934ULL), + QU(17903010316654728010ULL), QU( 9682617699648434744ULL), + QU(14757681836838592850ULL), QU( 1327242446558524473ULL), + QU(11126645098780572792ULL), QU( 1883602329313221774ULL), + QU( 2543897783922776873ULL), QU(15029168513767772842ULL), + QU(12710270651039129878ULL), QU(16118202956069604504ULL), + QU(15010759372168680524ULL), QU( 2296827082251923948ULL), + QU(10793729742623518101ULL), QU(13829764151845413046ULL), + QU(17769301223184451213ULL), QU( 3118268169210783372ULL), + QU(17626204544105123127ULL), QU( 7416718488974352644ULL), + QU(10450751996212925994ULL), QU( 9352529519128770586ULL), + QU( 259347569641110140ULL), QU( 8048588892269692697ULL), + QU( 1774414152306494058ULL), QU(10669548347214355622ULL), + QU(13061992253816795081ULL), QU(18432677803063861659ULL), + QU( 8879191055593984333ULL), QU(12433753195199268041ULL), + QU(14919392415439730602ULL), QU( 6612848378595332963ULL), + QU( 6320986812036143628ULL), QU(10465592420226092859ULL), + QU( 4196009278962570808ULL), QU( 3747816564473572224ULL), + QU(17941203486133732898ULL), QU( 2350310037040505198ULL), + QU( 5811779859134370113ULL), QU(10492109599506195126ULL), + QU( 7699650690179541274ULL), QU( 1954338494306022961ULL), + QU(14095816969027231152ULL), QU( 5841346919964852061ULL), + QU(14945969510148214735ULL), QU( 3680200305887550992ULL), + QU( 6218047466131695792ULL), QU( 8242165745175775096ULL), + QU(11021371934053307357ULL), QU( 1265099502753169797ULL), + QU( 4644347436111321718ULL), QU( 3609296916782832859ULL), + QU( 8109807992218521571ULL), QU(18387884215648662020ULL), + QU(14656324896296392902ULL), QU(17386819091238216751ULL), + QU(17788300878582317152ULL), QU( 7919446259742399591ULL), + QU( 4466613134576358004ULL), QU(12928181023667938509ULL), + QU(13147446154454932030ULL), QU(16552129038252734620ULL), + QU( 8395299403738822450ULL), QU(11313817655275361164ULL), + QU( 434258809499511718ULL), QU( 2074882104954788676ULL), + QU( 7929892178759395518ULL), QU( 9006461629105745388ULL), + QU( 5176475650000323086ULL), QU(11128357033468341069ULL), + QU(12026158851559118955ULL), QU(14699716249471156500ULL), + QU( 448982497120206757ULL), QU( 4156475356685519900ULL), + QU( 6063816103417215727ULL), QU(10073289387954971479ULL), + QU( 8174466846138590962ULL), QU( 2675777452363449006ULL), + QU( 9090685420572474281ULL), QU( 6659652652765562060ULL), + QU(12923120304018106621ULL), QU(11117480560334526775ULL), + QU( 937910473424587511ULL), QU( 1838692113502346645ULL), + QU(11133914074648726180ULL), QU( 7922600945143884053ULL), + QU(13435287702700959550ULL), QU( 5287964921251123332ULL), + QU(11354875374575318947ULL), QU(17955724760748238133ULL), + QU(13728617396297106512ULL), QU( 4107449660118101255ULL), + QU( 1210269794886589623ULL), QU(11408687205733456282ULL), + QU( 4538354710392677887ULL), QU(13566803319341319267ULL), + QU(17870798107734050771ULL), QU( 3354318982568089135ULL), + QU( 9034450839405133651ULL), QU(13087431795753424314ULL), + QU( 950333102820688239ULL), QU( 1968360654535604116ULL), + QU(16840551645563314995ULL), QU( 8867501803892924995ULL), + QU(11395388644490626845ULL), QU( 1529815836300732204ULL), + QU(13330848522996608842ULL), QU( 1813432878817504265ULL), + QU( 2336867432693429560ULL), QU(15192805445973385902ULL), + QU( 2528593071076407877ULL), QU( 128459777936689248ULL), + QU( 9976345382867214866ULL), QU( 6208885766767996043ULL), + QU(14982349522273141706ULL), QU( 3099654362410737822ULL), + QU(13776700761947297661ULL), QU( 8806185470684925550ULL), + QU( 8151717890410585321ULL), QU( 640860591588072925ULL), + QU(14592096303937307465ULL), QU( 9056472419613564846ULL), + QU(14861544647742266352ULL), QU(12703771500398470216ULL), + QU( 3142372800384138465ULL), QU( 6201105606917248196ULL), + QU(18337516409359270184ULL), QU(15042268695665115339ULL), + QU(15188246541383283846ULL), QU(12800028693090114519ULL), + QU( 5992859621101493472ULL), QU(18278043971816803521ULL), + QU( 9002773075219424560ULL), QU( 7325707116943598353ULL), + QU( 7930571931248040822ULL), QU( 5645275869617023448ULL), + QU( 7266107455295958487ULL), QU( 4363664528273524411ULL), + QU(14313875763787479809ULL), QU(17059695613553486802ULL), + QU( 9247761425889940932ULL), QU(13704726459237593128ULL), + QU( 2701312427328909832ULL), QU(17235532008287243115ULL), + QU(14093147761491729538ULL), QU( 6247352273768386516ULL), + QU( 8268710048153268415ULL), QU( 7985295214477182083ULL), + QU(15624495190888896807ULL), QU( 3772753430045262788ULL), + QU( 9133991620474991698ULL), QU( 5665791943316256028ULL), + QU( 7551996832462193473ULL), QU(13163729206798953877ULL), + QU( 9263532074153846374ULL), QU( 1015460703698618353ULL), + QU(17929874696989519390ULL), QU(18257884721466153847ULL), + QU(16271867543011222991ULL), QU( 3905971519021791941ULL), + QU(16814488397137052085ULL), QU( 1321197685504621613ULL), + QU( 2870359191894002181ULL), QU(14317282970323395450ULL), + QU(13663920845511074366ULL), QU( 2052463995796539594ULL), + QU(14126345686431444337ULL), QU( 1727572121947022534ULL), + QU(17793552254485594241ULL), QU( 6738857418849205750ULL), + QU( 1282987123157442952ULL), QU(16655480021581159251ULL), + QU( 6784587032080183866ULL), QU(14726758805359965162ULL), + QU( 7577995933961987349ULL), QU(12539609320311114036ULL), + QU(10789773033385439494ULL), QU( 8517001497411158227ULL), + QU(10075543932136339710ULL), QU(14838152340938811081ULL), + QU( 9560840631794044194ULL), QU(17445736541454117475ULL), + QU(10633026464336393186ULL), QU(15705729708242246293ULL), + QU( 1117517596891411098ULL), QU( 4305657943415886942ULL), + QU( 4948856840533979263ULL), QU(16071681989041789593ULL), + QU(13723031429272486527ULL), QU( 7639567622306509462ULL), + QU(12670424537483090390ULL), QU( 9715223453097197134ULL), + QU( 5457173389992686394ULL), QU( 289857129276135145ULL), + QU(17048610270521972512ULL), QU( 692768013309835485ULL), + QU(14823232360546632057ULL), QU(18218002361317895936ULL), + QU( 3281724260212650204ULL), QU(16453957266549513795ULL), + QU( 8592711109774511881ULL), QU( 929825123473369579ULL), + QU(15966784769764367791ULL), QU( 9627344291450607588ULL), + QU(10849555504977813287ULL), QU( 9234566913936339275ULL), + QU( 6413807690366911210ULL), QU(10862389016184219267ULL), + QU(13842504799335374048ULL), QU( 1531994113376881174ULL), + QU( 2081314867544364459ULL), QU(16430628791616959932ULL), + QU( 8314714038654394368ULL), QU( 9155473892098431813ULL), + QU(12577843786670475704ULL), QU( 4399161106452401017ULL), + QU( 1668083091682623186ULL), QU( 1741383777203714216ULL), + QU( 2162597285417794374ULL), QU(15841980159165218736ULL), + QU( 1971354603551467079ULL), QU( 1206714764913205968ULL), + QU( 4790860439591272330ULL), QU(14699375615594055799ULL), + QU( 8374423871657449988ULL), QU(10950685736472937738ULL), + QU( 697344331343267176ULL), QU(10084998763118059810ULL), + QU(12897369539795983124ULL), QU(12351260292144383605ULL), + QU( 1268810970176811234ULL), QU( 7406287800414582768ULL), + QU( 516169557043807831ULL), QU( 5077568278710520380ULL), + QU( 3828791738309039304ULL), QU( 7721974069946943610ULL), + QU( 3534670260981096460ULL), QU( 4865792189600584891ULL), + QU(16892578493734337298ULL), QU( 9161499464278042590ULL), + QU(11976149624067055931ULL), QU(13219479887277343990ULL), + QU(14161556738111500680ULL), QU(14670715255011223056ULL), + QU( 4671205678403576558ULL), QU(12633022931454259781ULL), + QU(14821376219869187646ULL), QU( 751181776484317028ULL), + QU( 2192211308839047070ULL), QU(11787306362361245189ULL), + QU(10672375120744095707ULL), QU( 4601972328345244467ULL), + QU(15457217788831125879ULL), QU( 8464345256775460809ULL), + QU(10191938789487159478ULL), QU( 6184348739615197613ULL), + QU(11425436778806882100ULL), QU( 2739227089124319793ULL), + QU( 461464518456000551ULL), QU( 4689850170029177442ULL), + QU( 6120307814374078625ULL), QU(11153579230681708671ULL), + QU( 7891721473905347926ULL), QU(10281646937824872400ULL), + QU( 3026099648191332248ULL), QU( 8666750296953273818ULL), + QU(14978499698844363232ULL), QU(13303395102890132065ULL), + QU( 8182358205292864080ULL), QU(10560547713972971291ULL), + QU(11981635489418959093ULL), QU( 3134621354935288409ULL), + QU(11580681977404383968ULL), QU(14205530317404088650ULL), + QU( 5997789011854923157ULL), QU(13659151593432238041ULL), + QU(11664332114338865086ULL), QU( 7490351383220929386ULL), + QU( 7189290499881530378ULL), QU(15039262734271020220ULL), + QU( 2057217285976980055ULL), QU( 555570804905355739ULL), + QU(11235311968348555110ULL), QU(13824557146269603217ULL), + QU(16906788840653099693ULL), QU( 7222878245455661677ULL), + QU( 5245139444332423756ULL), QU( 4723748462805674292ULL), + QU(12216509815698568612ULL), QU(17402362976648951187ULL), + QU(17389614836810366768ULL), QU( 4880936484146667711ULL), + QU( 9085007839292639880ULL), QU(13837353458498535449ULL), + QU(11914419854360366677ULL), QU(16595890135313864103ULL), + QU( 6313969847197627222ULL), QU(18296909792163910431ULL), + QU(10041780113382084042ULL), QU( 2499478551172884794ULL), + QU(11057894246241189489ULL), QU( 9742243032389068555ULL), + QU(12838934582673196228ULL), QU(13437023235248490367ULL), + QU(13372420669446163240ULL), QU( 6752564244716909224ULL), + QU( 7157333073400313737ULL), QU(12230281516370654308ULL), + QU( 1182884552219419117ULL), QU( 2955125381312499218ULL), + QU(10308827097079443249ULL), QU( 1337648572986534958ULL), + QU(16378788590020343939ULL), QU( 108619126514420935ULL), + QU( 3990981009621629188ULL), QU( 5460953070230946410ULL), + QU( 9703328329366531883ULL), QU(13166631489188077236ULL), + QU( 1104768831213675170ULL), QU( 3447930458553877908ULL), + QU( 8067172487769945676ULL), QU( 5445802098190775347ULL), + QU( 3244840981648973873ULL), QU(17314668322981950060ULL), + QU( 5006812527827763807ULL), QU(18158695070225526260ULL), + QU( 2824536478852417853ULL), QU(13974775809127519886ULL), + QU( 9814362769074067392ULL), QU(17276205156374862128ULL), + QU(11361680725379306967ULL), QU( 3422581970382012542ULL), + QU(11003189603753241266ULL), QU(11194292945277862261ULL), + QU( 6839623313908521348ULL), QU(11935326462707324634ULL), + QU( 1611456788685878444ULL), QU(13112620989475558907ULL), + QU( 517659108904450427ULL), QU(13558114318574407624ULL), + QU(15699089742731633077ULL), QU( 4988979278862685458ULL), + QU( 8111373583056521297ULL), QU( 3891258746615399627ULL), + QU( 8137298251469718086ULL), QU(12748663295624701649ULL), + QU( 4389835683495292062ULL), QU( 5775217872128831729ULL), + QU( 9462091896405534927ULL), QU( 8498124108820263989ULL), + QU( 8059131278842839525ULL), QU(10503167994254090892ULL), + QU(11613153541070396656ULL), QU(18069248738504647790ULL), + QU( 570657419109768508ULL), QU( 3950574167771159665ULL), + QU( 5514655599604313077ULL), QU( 2908460854428484165ULL), + QU(10777722615935663114ULL), QU(12007363304839279486ULL), + QU( 9800646187569484767ULL), QU( 8795423564889864287ULL), + QU(14257396680131028419ULL), QU( 6405465117315096498ULL), + QU( 7939411072208774878ULL), QU(17577572378528990006ULL), + QU(14785873806715994850ULL), QU(16770572680854747390ULL), + QU(18127549474419396481ULL), QU(11637013449455757750ULL), + QU(14371851933996761086ULL), QU( 3601181063650110280ULL), + QU( 4126442845019316144ULL), QU(10198287239244320669ULL), + QU(18000169628555379659ULL), QU(18392482400739978269ULL), + QU( 6219919037686919957ULL), QU( 3610085377719446052ULL), + QU( 2513925039981776336ULL), QU(16679413537926716955ULL), + QU(12903302131714909434ULL), QU( 5581145789762985009ULL), + QU(12325955044293303233ULL), QU(17216111180742141204ULL), + QU( 6321919595276545740ULL), QU( 3507521147216174501ULL), + QU( 9659194593319481840ULL), QU(11473976005975358326ULL), + QU(14742730101435987026ULL), QU( 492845897709954780ULL), + QU(16976371186162599676ULL), QU(17712703422837648655ULL), + QU( 9881254778587061697ULL), QU( 8413223156302299551ULL), + QU( 1563841828254089168ULL), QU( 9996032758786671975ULL), + QU( 138877700583772667ULL), QU(13003043368574995989ULL), + QU( 4390573668650456587ULL), QU( 8610287390568126755ULL), + QU(15126904974266642199ULL), QU( 6703637238986057662ULL), + QU( 2873075592956810157ULL), QU( 6035080933946049418ULL), + QU(13382846581202353014ULL), QU( 7303971031814642463ULL), + QU(18418024405307444267ULL), QU( 5847096731675404647ULL), + QU( 4035880699639842500ULL), QU(11525348625112218478ULL), + QU( 3041162365459574102ULL), QU( 2604734487727986558ULL), + QU(15526341771636983145ULL), QU(14556052310697370254ULL), + QU(12997787077930808155ULL), QU( 9601806501755554499ULL), + QU(11349677952521423389ULL), QU(14956777807644899350ULL), + QU(16559736957742852721ULL), QU(12360828274778140726ULL), + QU( 6685373272009662513ULL), QU(16932258748055324130ULL), + QU(15918051131954158508ULL), QU( 1692312913140790144ULL), + QU( 546653826801637367ULL), QU( 5341587076045986652ULL), + QU(14975057236342585662ULL), QU(12374976357340622412ULL), + QU(10328833995181940552ULL), QU(12831807101710443149ULL), + QU(10548514914382545716ULL), QU( 2217806727199715993ULL), + QU(12627067369242845138ULL), QU( 4598965364035438158ULL), + QU( 150923352751318171ULL), QU(14274109544442257283ULL), + QU( 4696661475093863031ULL), QU( 1505764114384654516ULL), + QU(10699185831891495147ULL), QU( 2392353847713620519ULL), + QU( 3652870166711788383ULL), QU( 8640653276221911108ULL), + QU( 3894077592275889704ULL), QU( 4918592872135964845ULL), + QU(16379121273281400789ULL), QU(12058465483591683656ULL), + QU(11250106829302924945ULL), QU( 1147537556296983005ULL), + QU( 6376342756004613268ULL), QU(14967128191709280506ULL), + QU(18007449949790627628ULL), QU( 9497178279316537841ULL), + QU( 7920174844809394893ULL), QU(10037752595255719907ULL), + QU(15875342784985217697ULL), QU(15311615921712850696ULL), + QU( 9552902652110992950ULL), QU(14054979450099721140ULL), + QU( 5998709773566417349ULL), QU(18027910339276320187ULL), + QU( 8223099053868585554ULL), QU( 7842270354824999767ULL), + QU( 4896315688770080292ULL), QU(12969320296569787895ULL), + QU( 2674321489185759961ULL), QU( 4053615936864718439ULL), + QU(11349775270588617578ULL), QU( 4743019256284553975ULL), + QU( 5602100217469723769ULL), QU(14398995691411527813ULL), + QU( 7412170493796825470ULL), QU( 836262406131744846ULL), + QU( 8231086633845153022ULL), QU( 5161377920438552287ULL), + QU( 8828731196169924949ULL), QU(16211142246465502680ULL), + QU( 3307990879253687818ULL), QU( 5193405406899782022ULL), + QU( 8510842117467566693ULL), QU( 6070955181022405365ULL), + QU(14482950231361409799ULL), QU(12585159371331138077ULL), + QU( 3511537678933588148ULL), QU( 2041849474531116417ULL), + QU(10944936685095345792ULL), QU(18303116923079107729ULL), + QU( 2720566371239725320ULL), QU( 4958672473562397622ULL), + QU( 3032326668253243412ULL), QU(13689418691726908338ULL), + QU( 1895205511728843996ULL), QU( 8146303515271990527ULL), + QU(16507343500056113480ULL), QU( 473996939105902919ULL), + QU( 9897686885246881481ULL), QU(14606433762712790575ULL), + QU( 6732796251605566368ULL), QU( 1399778120855368916ULL), + QU( 935023885182833777ULL), QU(16066282816186753477ULL), + QU( 7291270991820612055ULL), QU(17530230393129853844ULL), + QU(10223493623477451366ULL), QU(15841725630495676683ULL), + QU(17379567246435515824ULL), QU( 8588251429375561971ULL), + QU(18339511210887206423ULL), QU(17349587430725976100ULL), + QU(12244876521394838088ULL), QU( 6382187714147161259ULL), + QU(12335807181848950831ULL), QU(16948885622305460665ULL), + QU(13755097796371520506ULL), QU(14806740373324947801ULL), + QU( 4828699633859287703ULL), QU( 8209879281452301604ULL), + QU(12435716669553736437ULL), QU(13970976859588452131ULL), + QU( 6233960842566773148ULL), QU(12507096267900505759ULL), + QU( 1198713114381279421ULL), QU(14989862731124149015ULL), + QU(15932189508707978949ULL), QU( 2526406641432708722ULL), + QU( 29187427817271982ULL), QU( 1499802773054556353ULL), + QU(10816638187021897173ULL), QU( 5436139270839738132ULL), + QU( 6659882287036010082ULL), QU( 2154048955317173697ULL), + QU(10887317019333757642ULL), QU(16281091802634424955ULL), + QU(10754549879915384901ULL), QU(10760611745769249815ULL), + QU( 2161505946972504002ULL), QU( 5243132808986265107ULL), + QU(10129852179873415416ULL), QU( 710339480008649081ULL), + QU( 7802129453068808528ULL), QU(17967213567178907213ULL), + QU(15730859124668605599ULL), QU(13058356168962376502ULL), + QU( 3701224985413645909ULL), QU(14464065869149109264ULL), + QU( 9959272418844311646ULL), QU(10157426099515958752ULL), + QU(14013736814538268528ULL), QU(17797456992065653951ULL), + QU(17418878140257344806ULL), QU(15457429073540561521ULL), + QU( 2184426881360949378ULL), QU( 2062193041154712416ULL), + QU( 8553463347406931661ULL), QU( 4913057625202871854ULL), + QU( 2668943682126618425ULL), QU(17064444737891172288ULL), + QU( 4997115903913298637ULL), QU(12019402608892327416ULL), + QU(17603584559765897352ULL), QU(11367529582073647975ULL), + QU( 8211476043518436050ULL), QU( 8676849804070323674ULL), + QU(18431829230394475730ULL), QU(10490177861361247904ULL), + QU( 9508720602025651349ULL), QU( 7409627448555722700ULL), + QU( 5804047018862729008ULL), QU(11943858176893142594ULL), + QU(11908095418933847092ULL), QU( 5415449345715887652ULL), + QU( 1554022699166156407ULL), QU( 9073322106406017161ULL), + QU( 7080630967969047082ULL), QU(18049736940860732943ULL), + QU(12748714242594196794ULL), QU( 1226992415735156741ULL), + QU(17900981019609531193ULL), QU(11720739744008710999ULL), + QU( 3006400683394775434ULL), QU(11347974011751996028ULL), + QU( 3316999628257954608ULL), QU( 8384484563557639101ULL), + QU(18117794685961729767ULL), QU( 1900145025596618194ULL), + QU(17459527840632892676ULL), QU( 5634784101865710994ULL), + QU( 7918619300292897158ULL), QU( 3146577625026301350ULL), + QU( 9955212856499068767ULL), QU( 1873995843681746975ULL), + QU( 1561487759967972194ULL), QU( 8322718804375878474ULL), + QU(11300284215327028366ULL), QU( 4667391032508998982ULL), + QU( 9820104494306625580ULL), QU(17922397968599970610ULL), + QU( 1784690461886786712ULL), QU(14940365084341346821ULL), + QU( 5348719575594186181ULL), QU(10720419084507855261ULL), + QU(14210394354145143274ULL), QU( 2426468692164000131ULL), + QU(16271062114607059202ULL), QU(14851904092357070247ULL), + QU( 6524493015693121897ULL), QU( 9825473835127138531ULL), + QU(14222500616268569578ULL), QU(15521484052007487468ULL), + QU(14462579404124614699ULL), QU(11012375590820665520ULL), + QU(11625327350536084927ULL), QU(14452017765243785417ULL), + QU( 9989342263518766305ULL), QU( 3640105471101803790ULL), + QU( 4749866455897513242ULL), QU(13963064946736312044ULL), + QU(10007416591973223791ULL), QU(18314132234717431115ULL), + QU( 3286596588617483450ULL), QU( 7726163455370818765ULL), + QU( 7575454721115379328ULL), QU( 5308331576437663422ULL), + QU(18288821894903530934ULL), QU( 8028405805410554106ULL), + QU(15744019832103296628ULL), QU( 149765559630932100ULL), + QU( 6137705557200071977ULL), QU(14513416315434803615ULL), + QU(11665702820128984473ULL), QU( 218926670505601386ULL), + QU( 6868675028717769519ULL), QU(15282016569441512302ULL), + QU( 5707000497782960236ULL), QU( 6671120586555079567ULL), + QU( 2194098052618985448ULL), QU(16849577895477330978ULL), + QU(12957148471017466283ULL), QU( 1997805535404859393ULL), + QU( 1180721060263860490ULL), QU(13206391310193756958ULL), + QU(12980208674461861797ULL), QU( 3825967775058875366ULL), + QU(17543433670782042631ULL), QU( 1518339070120322730ULL), + QU(16344584340890991669ULL), QU( 2611327165318529819ULL), + QU(11265022723283422529ULL), QU( 4001552800373196817ULL), + QU(14509595890079346161ULL), QU( 3528717165416234562ULL), + QU(18153222571501914072ULL), QU( 9387182977209744425ULL), + QU(10064342315985580021ULL), QU(11373678413215253977ULL), + QU( 2308457853228798099ULL), QU( 9729042942839545302ULL), + QU( 7833785471140127746ULL), QU( 6351049900319844436ULL), + QU(14454610627133496067ULL), QU(12533175683634819111ULL), + QU(15570163926716513029ULL), QU(13356980519185762498ULL) }; TEST_BEGIN(test_gen_rand_32) From 22bc570fba00c4dd04cb4962e219d4230f137a4c Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 21 May 2014 18:06:14 +0900 Subject: [PATCH 033/352] Move __func__ to jemalloc_internal_macros.h test/integration/aligned_alloc.c needs it. --- include/jemalloc/internal/jemalloc_internal.h.in | 1 - include/jemalloc/internal/jemalloc_internal_macros.h | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index c9462e52..d9bfadf0 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -46,7 +46,6 @@ typedef intptr_t ssize_t; # define PATH_MAX 1024 # define STDERR_FILENO 2 -# define __func__ __FUNCTION__ /* Disable warnings about deprecated system functions */ # pragma warning(disable: 4996) #else diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h index 4e239230..bb81e990 100644 --- a/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/include/jemalloc/internal/jemalloc_internal_macros.h @@ -49,3 +49,7 @@ #ifndef JEMALLOC_HAS_RESTRICT # define restrict #endif + +#ifdef _MSC_VER +# define __func__ __FUNCTION__ +#endif From affe009e3765384805a23d804152fbf04151b117 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 28 May 2014 08:10:12 +0900 Subject: [PATCH 034/352] Use a configure test to detect the form of malloc_usable_size in malloc.h --- configure.ac | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 57015d1d..04bb2947 100644 --- a/configure.ac +++ b/configure.ac @@ -258,7 +258,6 @@ dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the dnl definitions need to be seen before any headers are included, which is a pain dnl to make happen otherwise. default_munmap="1" -JEMALLOC_USABLE_SIZE_CONST="const" case "${host}" in *-*-darwin*) CFLAGS="$CFLAGS" @@ -286,7 +285,6 @@ case "${host}" in AC_DEFINE([JEMALLOC_HAS_ALLOCA_H]) AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ]) AC_DEFINE([JEMALLOC_THREADED_INIT], [ ]) - JEMALLOC_USABLE_SIZE_CONST="" default_munmap="0" ;; *-*-netbsd*) @@ -351,6 +349,22 @@ case "${host}" in abi="elf" ;; esac + +JEMALLOC_USABLE_SIZE_CONST=const +AC_CHECK_HEADERS([malloc.h], [ + AC_MSG_CHECKING([whether malloc_usable_size definition can use const argument]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM( + [#include + #include + size_t malloc_usable_size(const void *ptr); + ], + [])],[ + AC_MSG_RESULT([yes]) + ],[ + JEMALLOC_USABLE_SIZE_CONST= + AC_MSG_RESULT([no]) + ]) +]) AC_DEFINE_UNQUOTED([JEMALLOC_USABLE_SIZE_CONST], [$JEMALLOC_USABLE_SIZE_CONST]) AC_SUBST([abi]) AC_SUBST([RPATH]) From 12f74e680c1d53c8fe5323a4ff66877534dcadd3 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 28 May 2014 12:39:13 +0900 Subject: [PATCH 035/352] Move platform headers and tricks from jemalloc_internal.h.in to a new jemalloc_internal_decls.h header --- .../jemalloc/internal/jemalloc_internal.h.in | 53 +---------------- .../internal/jemalloc_internal_decls.h | 58 +++++++++++++++++++ .../internal/jemalloc_internal_macros.h | 4 -- test/include/test/jemalloc_test_defs.h.in | 1 + 4 files changed, 60 insertions(+), 56 deletions(-) create mode 100644 include/jemalloc/internal/jemalloc_internal_decls.h diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index d9bfadf0..cf20f1f9 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -1,59 +1,8 @@ #ifndef JEMALLOC_INTERNAL_H #define JEMALLOC_INTERNAL_H -#include -#ifdef _WIN32 -# include -# define ENOENT ERROR_PATH_NOT_FOUND -# define EINVAL ERROR_BAD_ARGUMENTS -# define EAGAIN ERROR_OUTOFMEMORY -# define EPERM ERROR_WRITE_FAULT -# define EFAULT ERROR_INVALID_ADDRESS -# define ENOMEM ERROR_NOT_ENOUGH_MEMORY -# undef ERANGE -# define ERANGE ERROR_INVALID_DATA -#else -# include -# include -# include -# if !defined(SYS_write) && defined(__NR_write) -# define SYS_write __NR_write -# endif -# include -# include -# include -#endif -#include - -#include -#ifndef SIZE_T_MAX -# define SIZE_T_MAX SIZE_MAX -#endif -#include -#include -#include -#include -#include -#include -#ifndef offsetof -# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) -#endif -#include -#include -#include -#include -#ifdef _MSC_VER -# include -typedef intptr_t ssize_t; -# define PATH_MAX 1024 -# define STDERR_FILENO 2 -/* Disable warnings about deprecated system functions */ -# pragma warning(disable: 4996) -#else -# include -#endif -#include #include "jemalloc_internal_defs.h" +#include "jemalloc/internal/jemalloc_internal_decls.h" #ifdef JEMALLOC_UTRACE #include diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h new file mode 100644 index 00000000..7775ab38 --- /dev/null +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -0,0 +1,58 @@ +#ifndef JEMALLOC_INTERNAL_DECLS_H +#define JEMALLOC_INTERNAL_DECLS_H + +#include +#ifdef _WIN32 +# include +# define ENOENT ERROR_PATH_NOT_FOUND +# define EINVAL ERROR_BAD_ARGUMENTS +# define EAGAIN ERROR_OUTOFMEMORY +# define EPERM ERROR_WRITE_FAULT +# define EFAULT ERROR_INVALID_ADDRESS +# define ENOMEM ERROR_NOT_ENOUGH_MEMORY +# undef ERANGE +# define ERANGE ERROR_INVALID_DATA +#else +# include +# include +# include +# if !defined(SYS_write) && defined(__NR_write) +# define SYS_write __NR_write +# endif +# include +# include +# include +#endif +#include + +#include +#ifndef SIZE_T_MAX +# define SIZE_T_MAX SIZE_MAX +#endif +#include +#include +#include +#include +#include +#include +#ifndef offsetof +# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) +#endif +#include +#include +#include +#include +#ifdef _MSC_VER +# include +typedef intptr_t ssize_t; +# define PATH_MAX 1024 +# define STDERR_FILENO 2 +# define __func__ __FUNCTION__ +/* Disable warnings about deprecated system functions */ +# pragma warning(disable: 4996) +#else +# include +#endif +#include + +#endif /* JEMALLOC_INTERNAL_H */ diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h index bb81e990..4e239230 100644 --- a/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/include/jemalloc/internal/jemalloc_internal_macros.h @@ -49,7 +49,3 @@ #ifndef JEMALLOC_HAS_RESTRICT # define restrict #endif - -#ifdef _MSC_VER -# define __func__ __FUNCTION__ -#endif diff --git a/test/include/test/jemalloc_test_defs.h.in b/test/include/test/jemalloc_test_defs.h.in index 18a9773d..aaaaec14 100644 --- a/test/include/test/jemalloc_test_defs.h.in +++ b/test/include/test/jemalloc_test_defs.h.in @@ -1,4 +1,5 @@ #include "jemalloc/internal/jemalloc_internal_defs.h" +#include "jemalloc/internal/jemalloc_internal_decls.h" /* For use by SFMT. */ #undef HAVE_SSE2 From 26246af977250a520194a1ced89cbc73ce218ca7 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 28 May 2014 13:14:46 +0900 Subject: [PATCH 036/352] Define INFINITY when it's not defined --- test/unit/math.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/unit/math.c b/test/unit/math.c index a1b288ea..ebec77a6 100644 --- a/test/unit/math.c +++ b/test/unit/math.c @@ -3,6 +3,12 @@ #define MAX_REL_ERR 1.0e-9 #define MAX_ABS_ERR 1.0e-9 +#include + +#ifndef INFINITY +#define INFINITY (DBL_MAX + DBL_MAX) +#endif + static bool double_eq_rel(double a, double b, double max_rel_err, double max_abs_err) { From 17767b5f2b195076a8b57f8489addabb1ee68009 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 28 May 2014 14:06:30 +0900 Subject: [PATCH 037/352] Correctly return exit code from thd_join on Windows --- test/src/thd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/src/thd.c b/test/src/thd.c index 233242a1..7e53625f 100644 --- a/test/src/thd.c +++ b/test/src/thd.c @@ -14,7 +14,8 @@ void thd_join(thd_t thd, void **ret) { - WaitForSingleObject(thd, INFINITE); + if (WaitForSingleObject(thd, INFINITE) == WAIT_OBJECT_0 && ret) + GetExitCodeThread(thd, (LPDWORD) ret); } #else From b54aef1d8cc16f7b3f295cf857842aa6d5844d46 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 28 May 2014 14:17:01 +0900 Subject: [PATCH 038/352] Fixup after 3a730df (Avoid pointer arithmetic on void*[...]) --- test/integration/rallocx.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/rallocx.c b/test/integration/rallocx.c index e78e02f3..b6980729 100644 --- a/test/integration/rallocx.c +++ b/test/integration/rallocx.c @@ -95,7 +95,7 @@ TEST_BEGIN(test_zero) "Expected zeroed memory"); } if (psz != qsz) { - memset((void *)(uintptr_t)q+psz, FILL_BYTE, + memset((void *)((uintptr_t)q+psz), FILL_BYTE, qsz-psz); psz = qsz; } @@ -161,7 +161,7 @@ TEST_BEGIN(test_lg_align_and_zero) assert_false(validate_fill(q, 0, 0, MAX_VALIDATE), "Expected zeroed memory"); assert_false(validate_fill( - (void *)(uintptr_t)q+sz-MAX_VALIDATE, + (void *)((uintptr_t)q+sz-MAX_VALIDATE), 0, 0, MAX_VALIDATE), "Expected zeroed memory"); } p = q; From ccf046659a7c83e4e1573a1df30415144b4efdb6 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Wed, 7 May 2014 01:17:05 -0400 Subject: [PATCH 039/352] STATIC_PAGE_SHIFT for cross-compiling jemalloc Sets `STATIC_PAGE_SHIFT` for cross-compiling jemalloc to 12. A shift of 12 represents a page size of 4k for practically all platforms. --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 04bb2947..58f6289d 100644 --- a/configure.ac +++ b/configure.ac @@ -968,7 +968,8 @@ AC_CACHE_CHECK([STATIC_PAGE_SHIFT], return 0; ]])], [je_cv_static_page_shift=`cat conftest.out`], - [je_cv_static_page_shift=undefined])) + [je_cv_static_page_shift=undefined], + [je_cv_static_page_shift=12])) if test "x$je_cv_static_page_shift" != "xundefined"; then AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$je_cv_static_page_shift]) From 26f44df742893306a53a90328e15a62ed11b9e57 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 28 May 2014 11:08:17 -0700 Subject: [PATCH 040/352] Make sure initialization occurs prior to running tests. --- test/src/test.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/src/test.c b/test/src/test.c index 3acf8454..17728ca8 100644 --- a/test/src/test.c +++ b/test/src/test.c @@ -63,9 +63,22 @@ p_test_fini(void) test_status_t p_test(test_t *t, ...) { - test_status_t ret = test_status_pass; + test_status_t ret; va_list ap; + /* + * Make sure initialization occurs prior to running tests. Tests are + * special because they may use internal facilities prior to triggering + * initialization as a side effect of calling into the public API. This + * is a final safety that works even if jemalloc_constructor() doesn't + * run, as for MSVC builds. + */ + if (mallctl("version", NULL, NULL, NULL, 0) != 0) { + malloc_printf("Initialization error"); + return (test_status_fail); + } + + ret = test_status_pass; va_start(ap, t); for (; t != NULL; t = va_arg(ap, test_t *)) { t(); From 99118622ff5204feaabd2ee4109a7847ab388282 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 28 May 2014 11:23:01 -0700 Subject: [PATCH 041/352] Use nallocx() rather than mallctl() to trigger initialization. Use nallocx() rather than mallctl() to trigger initialization, because nallocx() has no side effects other than initialization, whereas mallctl() does a bunch of internal memory allocation. --- test/src/test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test.c b/test/src/test.c index 17728ca8..0f8bd494 100644 --- a/test/src/test.c +++ b/test/src/test.c @@ -73,7 +73,7 @@ p_test(test_t *t, ...) * is a final safety that works even if jemalloc_constructor() doesn't * run, as for MSVC builds. */ - if (mallctl("version", NULL, NULL, NULL, 0) != 0) { + if (nallocx(1, 0) == 0) { malloc_printf("Initialization error"); return (test_status_fail); } From d04047cc29bbc9d1f87a9346d1601e3dd87b6ca0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 28 May 2014 16:11:55 -0700 Subject: [PATCH 042/352] Add size class computation capability. Add size class computation capability, currently used only as validation of the size class lookup tables. Generalize the size class spacing used for bins, for eventual use throughout the full range of allocation sizes. --- configure.ac | 23 ++ include/jemalloc/internal/arena.h | 137 ++++++++- .../jemalloc/internal/jemalloc_internal.h.in | 4 +- .../internal/jemalloc_internal_defs.h.in | 5 + .../internal/jemalloc_internal_macros.h | 6 + include/jemalloc/internal/private_symbols.txt | 8 + include/jemalloc/internal/size_classes.sh | 261 ++++++++++++++---- include/jemalloc/internal/util.h | 47 ++++ src/arena.c | 62 +++-- 9 files changed, 462 insertions(+), 91 deletions(-) diff --git a/configure.ac b/configure.ac index 58f6289d..58522499 100644 --- a/configure.ac +++ b/configure.ac @@ -1200,6 +1200,29 @@ if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8) fi +dnl ============================================================================ +dnl Check for __builtin_clz() and __builtin_clzl(). + +AC_CACHE_CHECK([for __builtin_clz], + [je_cv_builtin_clz], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([], + [ + { + unsigned x = 0; + int y = __builtin_clz(x); + } + { + unsigned long x = 0; + int y = __builtin_clzl(x); + } + ])], + [je_cv_builtin_clz=yes], + [je_cv_builtin_clz=no])]) + +if test "x${je_cv_builtin_clz}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_BUILTIN_CLZ], [ ]) +fi + dnl ============================================================================ dnl Check for spinlock(3) operations as provided on Darwin. diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 598a89b0..2dc9501d 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -463,8 +463,15 @@ void arena_postfork_child(arena_t *arena); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +size_t small_size2bin_compute(size_t size); +size_t small_size2bin_lookup(size_t size); size_t small_size2bin(size_t size); +size_t small_bin2size_compute(size_t binind); +size_t small_bin2size_lookup(size_t binind); size_t small_bin2size(size_t binind); +size_t small_s2u_compute(size_t size); +size_t small_s2u_lookup(size_t size); +size_t small_s2u(size_t size); arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbitsp_read(size_t *mapbitsp); @@ -507,18 +514,144 @@ void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) # ifdef JEMALLOC_ARENA_INLINE_A +JEMALLOC_INLINE size_t +small_size2bin_compute(size_t size) +{ +#if (NTBINS != 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + size_t lg_ceil = lg_floor(pow2_ceil(size)); + return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); + } else +#endif + { + size_t x = lg_floor((size<<1)-1); + size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : + x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); + size_t grp = shift << LG_SIZE_CLASS_GROUP; + + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + + size_t delta_inverse_mask = ZI(-1) << lg_delta; + size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + size_t bin = NTBINS + grp + mod; + return (bin); + } +} + +JEMALLOC_ALWAYS_INLINE size_t +small_size2bin_lookup(size_t size) +{ + + assert(size <= LOOKUP_MAXCLASS); + { + size_t ret = ((size_t)(small_size2bin_tab[(size-1) >> + LG_TINY_MIN])); + assert(ret == small_size2bin_compute(size)); + return (ret); + } +} + JEMALLOC_ALWAYS_INLINE size_t small_size2bin(size_t size) { - return ((size_t)(small_size2bin_tab[(size-1) >> LG_TINY_MIN])); + assert(size > 0); + if (size <= LOOKUP_MAXCLASS) + return (small_size2bin_lookup(size)); + else + return (small_size2bin_compute(size)); +} + +JEMALLOC_INLINE size_t +small_bin2size_compute(size_t binind) +{ +#if (NTBINS > 0) + if (binind < NTBINS) + return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + binind)); + else +#endif + { + size_t reduced_binind = binind - NTBINS; + size_t grp = reduced_binind >> LG_SIZE_CLASS_GROUP; + size_t mod = reduced_binind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - + 1); + + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_QUANTUM + + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_QUANTUM-1); + size_t mod_size = (mod+1) << lg_delta; + + size_t usize = grp_size + mod_size; + return (usize); + } +} + +JEMALLOC_ALWAYS_INLINE size_t +small_bin2size_lookup(size_t binind) +{ + + assert(binind < NBINS); + { + size_t ret = ((size_t)(small_bin2size_tab[binind])); + assert(ret == small_bin2size_compute(binind)); + return (ret); + } } JEMALLOC_ALWAYS_INLINE size_t small_bin2size(size_t binind) { - return ((size_t)(small_bin2size_tab[binind])); + return (small_bin2size_lookup(binind)); +} + +JEMALLOC_ALWAYS_INLINE size_t +small_s2u_compute(size_t size) +{ +#if (NTBINS > 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + size_t lg_ceil = lg_floor(pow2_ceil(size)); + return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : + (ZU(1) << lg_ceil)); + } else +#endif + { + size_t x = lg_floor((size<<1)-1); + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (size + delta_mask) & ~delta_mask; + return (usize); + } +} + +JEMALLOC_ALWAYS_INLINE size_t +small_s2u_lookup(size_t size) +{ + size_t ret = (small_bin2size(small_size2bin(size))); + + assert(ret == small_s2u_compute(size)); + return (ret); +} + +JEMALLOC_ALWAYS_INLINE size_t +small_s2u(size_t size) +{ + + assert(size > 0); + if (size <= LOOKUP_MAXCLASS) + return (small_s2u_lookup(size)); + else + return (small_s2u_compute(size)); } # endif /* JEMALLOC_ARENA_INLINE_A */ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index cf20f1f9..491345c9 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -475,7 +475,7 @@ s2u(size_t size) { if (size <= SMALL_MAXCLASS) - return (small_bin2size(small_size2bin(size))); + return (small_s2u(size)); if (size <= arena_maxclass) return (PAGE_CEILING(size)); return (CHUNK_CEILING(size)); @@ -518,7 +518,7 @@ sa2u(size_t size, size_t alignment) if (usize <= arena_maxclass && alignment <= PAGE) { if (usize <= SMALL_MAXCLASS) - return (small_bin2size(small_size2bin(usize))); + return (small_s2u(usize)); return (PAGE_CEILING(usize)); } else { size_t run_size; diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 09ddd4f3..a9a50f14 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -47,6 +47,11 @@ */ #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 +/* + * Defined if __builtin_clz() and __builtin_clzl() are available. + */ +#undef JEMALLOC_HAVE_BUILTIN_CLZ + /* * Defined if OSSpin*() functions are available, as provided by Darwin, and * documented in the spinlock(3) manual page. diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h index 4e239230..38e28861 100644 --- a/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/include/jemalloc/internal/jemalloc_internal_macros.h @@ -39,9 +39,15 @@ #endif #define ZU(z) ((size_t)z) +#define ZI(z) ((ssize_t)z) #define QU(q) ((uint64_t)q) #define QI(q) ((int64_t)q) +#define KZU(z) ZU(z##ULL) +#define KZI(z) ZI(z##ULL) +#define KQU(q) QU(q##ULL) +#define KQI(q) QI(q##ULL) + #ifndef __DECONST # define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) #endif diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index f6c4fbcc..3401301c 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -234,6 +234,7 @@ ixalloc jemalloc_postfork_child jemalloc_postfork_parent jemalloc_prefork +lg_floor malloc_cprintf malloc_mutex_init malloc_mutex_lock @@ -348,8 +349,15 @@ s2u sa2u set_errno small_bin2size +small_bin2size_compute +small_bin2size_lookup small_bin2size_tab +small_s2u +small_s2u_compute +small_s2u_lookup small_size2bin +small_size2bin_compute +small_size2bin_lookup small_size2bin_tab stats_cactive stats_cactive_add diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 960674aa..3edebf23 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -2,16 +2,23 @@ # The following limits are chosen such that they cover all supported platforms. -# Range of quanta. -lg_qmin=3 -lg_qmax=4 +# Pointer sizes. +lg_zarr="2 3" + +# Quanta. +lg_qarr="3 4" # The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)]. lg_tmin=3 -# Range of page sizes. -lg_pmin=12 -lg_pmax=16 +# Maximum lookup size. +lg_kmax=12 + +# Page sizes. +lg_parr="12 13 16" + +# Size class group size (number of size classes for each size doubling). +lg_g=2 pow2() { e=$1 @@ -22,68 +29,206 @@ pow2() { done } +lg() { + x=$1 + lg_result=0 + while [ ${x} -gt 1 ] ; do + lg_result=$((${lg_result} + 1)) + x=$((${x} / 2)) + done +} + +size_class() { + index=$1 + lg_grp=$2 + lg_delta=$3 + ndelta=$4 + lg_p=$5 + lg_kmax=$6 + + lg ${ndelta}; lg_ndelta=${lg_result}; pow2 ${lg_ndelta} + if [ ${pow2_result} -lt ${ndelta} ] ; then + rem="yes" + else + rem="no" + fi + + lg_size=${lg_grp} + if [ $((${lg_delta} + ${lg_ndelta})) -eq ${lg_grp} ] ; then + lg_size=$((${lg_grp} + 1)) + else + lg_size=${lg_grp} + rem="yes" + fi + + if [ ${lg_size} -lt ${lg_p} ] ; then + bin="yes" + else + bin="no" + fi + if [ ${lg_size} -lt ${lg_kmax} \ + -o ${lg_size} -eq ${lg_kmax} -a ${rem} = "no" ] ; then + lg_delta_lookup=${lg_delta} + else + lg_delta_lookup="no" + fi + printf ' SC(%3d, %6d, %8d, %6d, %3s, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${bin} ${lg_delta_lookup} + # Defined upon return: + # - lg_delta_lookup (${lg_delta} or "no") + # - bin ("yes" or "no") +} + +sep_line() { + echo " \\" +} + +size_classes() { + lg_z=$1 + lg_q=$2 + lg_t=$3 + lg_p=$4 + lg_g=$5 + + pow2 $((${lg_z} + 3)); ptr_bits=${pow2_result} + pow2 ${lg_g}; g=${pow2_result} + + echo "#define SIZE_CLASSES \\" + echo " /* index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup */ \\" + + ntbins=0 + nlbins=0 + lg_tiny_maxclass='"NA"' + nbins=0 + + # Tiny size classes. + ndelta=0 + index=0 + lg_grp=${lg_t} + lg_delta=${lg_grp} + while [ ${lg_grp} -lt ${lg_q} ] ; do + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + if [ ${lg_delta_lookup} != "no" ] ; then + nlbins=$((${index} + 1)) + fi + if [ ${bin} != "no" ] ; then + nbins=$((${index} + 1)) + fi + ntbins=$((${ntbins} + 1)) + lg_tiny_maxclass=${lg_grp} # Final written value is correct. + index=$((${index} + 1)) + lg_delta=${lg_grp} + lg_grp=$((${lg_grp} + 1)) + done + + # First non-tiny group. + if [ ${ntbins} -gt 0 ] ; then + sep_line + # The first size class has an unusual encoding, because the size has to be + # split between grp and delta*ndelta. + lg_grp=$((${lg_grp} - 1)) + ndelta=1 + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + index=$((${index} + 1)) + lg_grp=$((${lg_grp} + 1)) + lg_delta=$((${lg_delta} + 1)) + fi + while [ ${ndelta} -lt ${g} ] ; do + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + index=$((${index} + 1)) + ndelta=$((${ndelta} + 1)) + done + + # All remaining groups. + lg_grp=$((${lg_grp} + ${lg_g})) + while [ ${lg_grp} -lt ${ptr_bits} ] ; do + sep_line + ndelta=1 + if [ ${lg_grp} -eq $((${ptr_bits} - 1)) ] ; then + ndelta_limit=$((${g} - 1)) + else + ndelta_limit=${g} + fi + while [ ${ndelta} -le ${ndelta_limit} ] ; do + size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} + if [ ${lg_delta_lookup} != "no" ] ; then + nlbins=$((${index} + 1)) + # Final written value is correct: + lookup_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" + fi + if [ ${bin} != "no" ] ; then + nbins=$((${index} + 1)) + # Final written value is correct: + small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" + fi + index=$((${index} + 1)) + ndelta=$((${ndelta} + 1)) + done + lg_grp=$((${lg_grp} + 1)) + lg_delta=$((${lg_delta} + 1)) + done + echo + + # Defined upon completion: + # - ntbins + # - nlbins + # - nbins + # - lg_tiny_maxclass + # - lookup_maxclass + # - small_maxclass +} + cat <> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); +#if (LG_SIZEOF_PTR == 3 && LG_SIZEOF_PTR == LG_SIZEOF_LONG) + x |= (x >> 32); + return (65 - ffsl(~x)); +#elif (LG_SIZEOF_PTR == 2) + return (33 - ffs(~x)); +#else +# error "Unsupported type sizes for lg_floor()" +#endif +} +#endif + /* Sets error code */ JEMALLOC_INLINE void set_errno(int errnum) diff --git a/src/arena.c b/src/arena.c index f5d7d062..c392419e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -9,40 +9,39 @@ arena_bin_info_t arena_bin_info[NBINS]; JEMALLOC_ALIGNED(CACHELINE) const uint32_t small_bin2size_tab[NBINS] = { -#define SIZE_CLASS(bin, delta, size) \ +#define B2S_bin_yes(size) \ size, +#define B2S_bin_no(size) +#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ + B2S_bin_##bin((ZU(1)<reg_size = size; \ prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); +#define BIN_INFO_INIT_bin_no(index, size) +#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ + BIN_INFO_INIT_bin_##bin(index, (ZU(1)< Date: Wed, 28 May 2014 21:14:16 -0700 Subject: [PATCH 043/352] Use KQU() rather than QU() where applicable. Fix KZI() and KQI() to append LL rather than ULL. --- include/jemalloc/internal/hash.h | 8 +- .../internal/jemalloc_internal_macros.h | 4 +- test/src/SFMT.c | 2 +- test/unit/SFMT.c | 2000 ++++++++--------- test/unit/util.c | 78 +- 5 files changed, 1046 insertions(+), 1046 deletions(-) diff --git a/include/jemalloc/internal/hash.h b/include/jemalloc/internal/hash.h index f2b3a16c..a43bbbec 100644 --- a/include/jemalloc/internal/hash.h +++ b/include/jemalloc/internal/hash.h @@ -76,9 +76,9 @@ hash_fmix_64(uint64_t k) { k ^= k >> 33; - k *= QU(0xff51afd7ed558ccdULL); + k *= KQU(0xff51afd7ed558ccd); k ^= k >> 33; - k *= QU(0xc4ceb9fe1a85ec53ULL); + k *= KQU(0xc4ceb9fe1a85ec53); k ^= k >> 33; return (k); @@ -247,8 +247,8 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t h1 = seed; uint64_t h2 = seed; - const uint64_t c1 = QU(0x87c37b91114253d5ULL); - const uint64_t c2 = QU(0x4cf5ad432745937fULL); + const uint64_t c1 = KQU(0x87c37b91114253d5); + const uint64_t c2 = KQU(0x4cf5ad432745937f); /* body */ { diff --git a/include/jemalloc/internal/jemalloc_internal_macros.h b/include/jemalloc/internal/jemalloc_internal_macros.h index 38e28861..a08ba772 100644 --- a/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/include/jemalloc/internal/jemalloc_internal_macros.h @@ -44,9 +44,9 @@ #define QI(q) ((int64_t)q) #define KZU(z) ZU(z##ULL) -#define KZI(z) ZI(z##ULL) +#define KZI(z) ZI(z##LL) #define KQU(q) QU(q##ULL) -#define KQI(q) QI(q##ULL) +#define KQI(q) QI(q##LL) #ifndef __DECONST # define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) diff --git a/test/src/SFMT.c b/test/src/SFMT.c index d2cc9d1c..22a5ac55 100644 --- a/test/src/SFMT.c +++ b/test/src/SFMT.c @@ -511,7 +511,7 @@ uint64_t gen_rand64(sfmt_t *ctx) { uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit) { uint64_t ret, above; - above = 0xffffffffffffffffULL - (0xffffffffffffffffULL % limit); + above = KQU(0xffffffffffffffff) - (KQU(0xffffffffffffffff) % limit); while (1) { ret = gen_rand64(ctx); if (ret < above) { diff --git a/test/unit/SFMT.c b/test/unit/SFMT.c index 0ad9c233..88b31f6e 100644 --- a/test/unit/SFMT.c +++ b/test/unit/SFMT.c @@ -445,1008 +445,1008 @@ static const uint32_t init_by_array_32_expected[] = { 2750138839U, 3518055702U, 733072558U, 4169325400U, 788493625U }; static const uint64_t init_gen_rand_64_expected[] = { - QU(16924766246869039260ULL), QU( 8201438687333352714ULL), - QU( 2265290287015001750ULL), QU(18397264611805473832ULL), - QU( 3375255223302384358ULL), QU( 6345559975416828796ULL), - QU(18229739242790328073ULL), QU( 7596792742098800905ULL), - QU( 255338647169685981ULL), QU( 2052747240048610300ULL), - QU(18328151576097299343ULL), QU(12472905421133796567ULL), - QU(11315245349717600863ULL), QU(16594110197775871209ULL), - QU(15708751964632456450ULL), QU(10452031272054632535ULL), - QU(11097646720811454386ULL), QU( 4556090668445745441ULL), - QU(17116187693090663106ULL), QU(14931526836144510645ULL), - QU( 9190752218020552591ULL), QU( 9625800285771901401ULL), - QU(13995141077659972832ULL), QU( 5194209094927829625ULL), - QU( 4156788379151063303ULL), QU( 8523452593770139494ULL), - QU(14082382103049296727ULL), QU( 2462601863986088483ULL), - QU( 3030583461592840678ULL), QU( 5221622077872827681ULL), - QU( 3084210671228981236ULL), QU(13956758381389953823ULL), - QU(13503889856213423831ULL), QU(15696904024189836170ULL), - QU( 4612584152877036206ULL), QU( 6231135538447867881ULL), - QU(10172457294158869468ULL), QU( 6452258628466708150ULL), - QU(14044432824917330221ULL), QU( 370168364480044279ULL), - QU(10102144686427193359ULL), QU( 667870489994776076ULL), - QU( 2732271956925885858ULL), QU(18027788905977284151ULL), - QU(15009842788582923859ULL), QU( 7136357960180199542ULL), - QU(15901736243475578127ULL), QU(16951293785352615701ULL), - QU(10551492125243691632ULL), QU(17668869969146434804ULL), - QU(13646002971174390445ULL), QU( 9804471050759613248ULL), - QU( 5511670439655935493ULL), QU(18103342091070400926ULL), - QU(17224512747665137533ULL), QU(15534627482992618168ULL), - QU( 1423813266186582647ULL), QU(15821176807932930024ULL), - QU( 30323369733607156ULL), QU(11599382494723479403ULL), - QU( 653856076586810062ULL), QU( 3176437395144899659ULL), - QU(14028076268147963917ULL), QU(16156398271809666195ULL), - QU( 3166955484848201676ULL), QU( 5746805620136919390ULL), - QU(17297845208891256593ULL), QU(11691653183226428483ULL), - QU(17900026146506981577ULL), QU(15387382115755971042ULL), - QU(16923567681040845943ULL), QU( 8039057517199388606ULL), - QU(11748409241468629263ULL), QU( 794358245539076095ULL), - QU(13438501964693401242ULL), QU(14036803236515618962ULL), - QU( 5252311215205424721ULL), QU(17806589612915509081ULL), - QU( 6802767092397596006ULL), QU(14212120431184557140ULL), - QU( 1072951366761385712ULL), QU(13098491780722836296ULL), - QU( 9466676828710797353ULL), QU(12673056849042830081ULL), - QU(12763726623645357580ULL), QU(16468961652999309493ULL), - QU(15305979875636438926ULL), QU(17444713151223449734ULL), - QU( 5692214267627883674ULL), QU(13049589139196151505ULL), - QU( 880115207831670745ULL), QU( 1776529075789695498ULL), - QU(16695225897801466485ULL), QU(10666901778795346845ULL), - QU( 6164389346722833869ULL), QU( 2863817793264300475ULL), - QU( 9464049921886304754ULL), QU( 3993566636740015468ULL), - QU( 9983749692528514136ULL), QU(16375286075057755211ULL), - QU(16042643417005440820ULL), QU(11445419662923489877ULL), - QU( 7999038846885158836ULL), QU( 6721913661721511535ULL), - QU( 5363052654139357320ULL), QU( 1817788761173584205ULL), - QU(13290974386445856444ULL), QU( 4650350818937984680ULL), - QU( 8219183528102484836ULL), QU( 1569862923500819899ULL), - QU( 4189359732136641860ULL), QU(14202822961683148583ULL), - QU( 4457498315309429058ULL), QU(13089067387019074834ULL), - QU(11075517153328927293ULL), QU(10277016248336668389ULL), - QU( 7070509725324401122ULL), QU(17808892017780289380ULL), - QU(13143367339909287349ULL), QU( 1377743745360085151ULL), - QU( 5749341807421286485ULL), QU(14832814616770931325ULL), - QU( 7688820635324359492ULL), QU(10960474011539770045ULL), - QU( 81970066653179790ULL), QU(12619476072607878022ULL), - QU( 4419566616271201744ULL), QU(15147917311750568503ULL), - QU( 5549739182852706345ULL), QU( 7308198397975204770ULL), - QU(13580425496671289278ULL), QU(17070764785210130301ULL), - QU( 8202832846285604405ULL), QU( 6873046287640887249ULL), - QU( 6927424434308206114ULL), QU( 6139014645937224874ULL), - QU(10290373645978487639ULL), QU(15904261291701523804ULL), - QU( 9628743442057826883ULL), QU(18383429096255546714ULL), - QU( 4977413265753686967ULL), QU( 7714317492425012869ULL), - QU( 9025232586309926193ULL), QU(14627338359776709107ULL), - QU(14759849896467790763ULL), QU(10931129435864423252ULL), - QU( 4588456988775014359ULL), QU(10699388531797056724ULL), - QU( 468652268869238792ULL), QU( 5755943035328078086ULL), - QU( 2102437379988580216ULL), QU( 9986312786506674028ULL), - QU( 2654207180040945604ULL), QU( 8726634790559960062ULL), - QU( 100497234871808137ULL), QU( 2800137176951425819ULL), - QU( 6076627612918553487ULL), QU( 5780186919186152796ULL), - QU( 8179183595769929098ULL), QU( 6009426283716221169ULL), - QU( 2796662551397449358ULL), QU( 1756961367041986764ULL), - QU( 6972897917355606205ULL), QU(14524774345368968243ULL), - QU( 2773529684745706940ULL), QU( 4853632376213075959ULL), - QU( 4198177923731358102ULL), QU( 8271224913084139776ULL), - QU( 2741753121611092226ULL), QU(16782366145996731181ULL), - QU(15426125238972640790ULL), QU(13595497100671260342ULL), - QU( 3173531022836259898ULL), QU( 6573264560319511662ULL), - QU(18041111951511157441ULL), QU( 2351433581833135952ULL), - QU( 3113255578908173487ULL), QU( 1739371330877858784ULL), - QU(16046126562789165480ULL), QU( 8072101652214192925ULL), - QU(15267091584090664910ULL), QU( 9309579200403648940ULL), - QU( 5218892439752408722ULL), QU(14492477246004337115ULL), - QU(17431037586679770619ULL), QU( 7385248135963250480ULL), - QU( 9580144956565560660ULL), QU( 4919546228040008720ULL), - QU(15261542469145035584ULL), QU(18233297270822253102ULL), - QU( 5453248417992302857ULL), QU( 9309519155931460285ULL), - QU(10342813012345291756ULL), QU(15676085186784762381ULL), - QU(15912092950691300645ULL), QU( 9371053121499003195ULL), - QU( 9897186478226866746ULL), QU(14061858287188196327ULL), - QU( 122575971620788119ULL), QU(12146750969116317754ULL), - QU( 4438317272813245201ULL), QU( 8332576791009527119ULL), - QU(13907785691786542057ULL), QU(10374194887283287467ULL), - QU( 2098798755649059566ULL), QU( 3416235197748288894ULL), - QU( 8688269957320773484ULL), QU( 7503964602397371571ULL), - QU(16724977015147478236ULL), QU( 9461512855439858184ULL), - QU(13259049744534534727ULL), QU( 3583094952542899294ULL), - QU( 8764245731305528292ULL), QU(13240823595462088985ULL), - QU(13716141617617910448ULL), QU(18114969519935960955ULL), - QU( 2297553615798302206ULL), QU( 4585521442944663362ULL), - QU(17776858680630198686ULL), QU( 4685873229192163363ULL), - QU( 152558080671135627ULL), QU(15424900540842670088ULL), - QU(13229630297130024108ULL), QU(17530268788245718717ULL), - QU(16675633913065714144ULL), QU( 3158912717897568068ULL), - QU(15399132185380087288ULL), QU( 7401418744515677872ULL), - QU(13135412922344398535ULL), QU( 6385314346100509511ULL), - QU(13962867001134161139ULL), QU(10272780155442671999ULL), - QU(12894856086597769142ULL), QU(13340877795287554994ULL), - QU(12913630602094607396ULL), QU(12543167911119793857ULL), - QU(17343570372251873096ULL), QU(10959487764494150545ULL), - QU( 6966737953093821128ULL), QU(13780699135496988601ULL), - QU( 4405070719380142046ULL), QU(14923788365607284982ULL), - QU( 2869487678905148380ULL), QU( 6416272754197188403ULL), - QU(15017380475943612591ULL), QU( 1995636220918429487ULL), - QU( 3402016804620122716ULL), QU(15800188663407057080ULL), - QU(11362369990390932882ULL), QU(15262183501637986147ULL), - QU(10239175385387371494ULL), QU( 9352042420365748334ULL), - QU( 1682457034285119875ULL), QU( 1724710651376289644ULL), - QU( 2038157098893817966ULL), QU( 9897825558324608773ULL), - QU( 1477666236519164736ULL), QU(16835397314511233640ULL), - QU(10370866327005346508ULL), QU(10157504370660621982ULL), - QU(12113904045335882069ULL), QU(13326444439742783008ULL), - QU(11302769043000765804ULL), QU(13594979923955228484ULL), - QU(11779351762613475968ULL), QU( 3786101619539298383ULL), - QU( 8021122969180846063ULL), QU(15745904401162500495ULL), - QU(10762168465993897267ULL), QU(13552058957896319026ULL), - QU(11200228655252462013ULL), QU( 5035370357337441226ULL), - QU( 7593918984545500013ULL), QU( 5418554918361528700ULL), - QU( 4858270799405446371ULL), QU( 9974659566876282544ULL), - QU(18227595922273957859ULL), QU( 2772778443635656220ULL), - QU(14285143053182085385ULL), QU( 9939700992429600469ULL), - QU(12756185904545598068ULL), QU( 2020783375367345262ULL), - QU( 57026775058331227ULL), QU( 950827867930065454ULL), - QU( 6602279670145371217ULL), QU( 2291171535443566929ULL), - QU( 5832380724425010313ULL), QU( 1220343904715982285ULL), - QU(17045542598598037633ULL), QU(15460481779702820971ULL), - QU(13948388779949365130ULL), QU(13975040175430829518ULL), - QU(17477538238425541763ULL), QU(11104663041851745725ULL), - QU(15860992957141157587ULL), QU(14529434633012950138ULL), - QU( 2504838019075394203ULL), QU( 7512113882611121886ULL), - QU( 4859973559980886617ULL), QU( 1258601555703250219ULL), - QU(15594548157514316394ULL), QU( 4516730171963773048ULL), - QU(11380103193905031983ULL), QU( 6809282239982353344ULL), - QU(18045256930420065002ULL), QU( 2453702683108791859ULL), - QU( 977214582986981460ULL), QU( 2006410402232713466ULL), - QU( 6192236267216378358ULL), QU( 3429468402195675253ULL), - QU(18146933153017348921ULL), QU(17369978576367231139ULL), - QU( 1246940717230386603ULL), QU(11335758870083327110ULL), - QU(14166488801730353682ULL), QU( 9008573127269635732ULL), - QU(10776025389820643815ULL), QU(15087605441903942962ULL), - QU( 1359542462712147922ULL), QU(13898874411226454206ULL), - QU(17911176066536804411ULL), QU( 9435590428600085274ULL), - QU( 294488509967864007ULL), QU( 8890111397567922046ULL), - QU( 7987823476034328778ULL), QU(13263827582440967651ULL), - QU( 7503774813106751573ULL), QU(14974747296185646837ULL), - QU( 8504765037032103375ULL), QU(17340303357444536213ULL), - QU( 7704610912964485743ULL), QU( 8107533670327205061ULL), - QU( 9062969835083315985ULL), QU(16968963142126734184ULL), - QU(12958041214190810180ULL), QU( 2720170147759570200ULL), - QU( 2986358963942189566ULL), QU(14884226322219356580ULL), - QU( 286224325144368520ULL), QU(11313800433154279797ULL), - QU(18366849528439673248ULL), QU(17899725929482368789ULL), - QU( 3730004284609106799ULL), QU( 1654474302052767205ULL), - QU( 5006698007047077032ULL), QU( 8196893913601182838ULL), - QU(15214541774425211640ULL), QU(17391346045606626073ULL), - QU( 8369003584076969089ULL), QU( 3939046733368550293ULL), - QU(10178639720308707785ULL), QU( 2180248669304388697ULL), - QU( 62894391300126322ULL), QU( 9205708961736223191ULL), - QU( 6837431058165360438ULL), QU( 3150743890848308214ULL), - QU(17849330658111464583ULL), QU(12214815643135450865ULL), - QU(13410713840519603402ULL), QU( 3200778126692046802ULL), - QU(13354780043041779313ULL), QU( 800850022756886036ULL), - QU(15660052933953067433ULL), QU( 6572823544154375676ULL), - QU(11030281857015819266ULL), QU(12682241941471433835ULL), - QU(11654136407300274693ULL), QU( 4517795492388641109ULL), - QU( 9757017371504524244ULL), QU(17833043400781889277ULL), - QU(12685085201747792227ULL), QU(10408057728835019573ULL), - QU( 98370418513455221ULL), QU( 6732663555696848598ULL), - QU(13248530959948529780ULL), QU( 3530441401230622826ULL), - QU(18188251992895660615ULL), QU( 1847918354186383756ULL), - QU( 1127392190402660921ULL), QU(11293734643143819463ULL), - QU( 3015506344578682982ULL), QU(13852645444071153329ULL), - QU( 2121359659091349142ULL), QU( 1294604376116677694ULL), - QU( 5616576231286352318ULL), QU( 7112502442954235625ULL), - QU(11676228199551561689ULL), QU(12925182803007305359ULL), - QU( 7852375518160493082ULL), QU( 1136513130539296154ULL), - QU( 5636923900916593195ULL), QU( 3221077517612607747ULL), - QU(17784790465798152513ULL), QU( 3554210049056995938ULL), - QU(17476839685878225874ULL), QU( 3206836372585575732ULL), - QU( 2765333945644823430ULL), QU(10080070903718799528ULL), - QU( 5412370818878286353ULL), QU( 9689685887726257728ULL), - QU( 8236117509123533998ULL), QU( 1951139137165040214ULL), - QU( 4492205209227980349ULL), QU(16541291230861602967ULL), - QU( 1424371548301437940ULL), QU( 9117562079669206794ULL), - QU(14374681563251691625ULL), QU(13873164030199921303ULL), - QU( 6680317946770936731ULL), QU(15586334026918276214ULL), - QU(10896213950976109802ULL), QU( 9506261949596413689ULL), - QU( 9903949574308040616ULL), QU( 6038397344557204470ULL), - QU( 174601465422373648ULL), QU(15946141191338238030ULL), - QU(17142225620992044937ULL), QU( 7552030283784477064ULL), - QU( 2947372384532947997ULL), QU( 510797021688197711ULL), - QU( 4962499439249363461ULL), QU( 23770320158385357ULL), - QU( 959774499105138124ULL), QU( 1468396011518788276ULL), - QU( 2015698006852312308ULL), QU( 4149400718489980136ULL), - QU( 5992916099522371188ULL), QU(10819182935265531076ULL), - QU(16189787999192351131ULL), QU( 342833961790261950ULL), - QU(12470830319550495336ULL), QU(18128495041912812501ULL), - QU( 1193600899723524337ULL), QU( 9056793666590079770ULL), - QU( 2154021227041669041ULL), QU( 4963570213951235735ULL), - QU( 4865075960209211409ULL), QU( 2097724599039942963ULL), - QU( 2024080278583179845ULL), QU(11527054549196576736ULL), - QU(10650256084182390252ULL), QU( 4808408648695766755ULL), - QU( 1642839215013788844ULL), QU(10607187948250398390ULL), - QU( 7076868166085913508ULL), QU( 730522571106887032ULL), - QU(12500579240208524895ULL), QU( 4484390097311355324ULL), - QU(15145801330700623870ULL), QU( 8055827661392944028ULL), - QU( 5865092976832712268ULL), QU(15159212508053625143ULL), - QU( 3560964582876483341ULL), QU( 4070052741344438280ULL), - QU( 6032585709886855634ULL), QU(15643262320904604873ULL), - QU( 2565119772293371111ULL), QU( 318314293065348260ULL), - QU(15047458749141511872ULL), QU( 7772788389811528730ULL), - QU( 7081187494343801976ULL), QU( 6465136009467253947ULL), - QU(10425940692543362069ULL), QU( 554608190318339115ULL), - QU(14796699860302125214ULL), QU( 1638153134431111443ULL), - QU(10336967447052276248ULL), QU( 8412308070396592958ULL), - QU( 4004557277152051226ULL), QU( 8143598997278774834ULL), - QU(16413323996508783221ULL), QU(13139418758033994949ULL), - QU( 9772709138335006667ULL), QU( 2818167159287157659ULL), - QU(17091740573832523669ULL), QU(14629199013130751608ULL), - QU(18268322711500338185ULL), QU( 8290963415675493063ULL), - QU( 8830864907452542588ULL), QU( 1614839084637494849ULL), - QU(14855358500870422231ULL), QU( 3472996748392519937ULL), - QU(15317151166268877716ULL), QU( 5825895018698400362ULL), - QU(16730208429367544129ULL), QU(10481156578141202800ULL), - QU( 4746166512382823750ULL), QU(12720876014472464998ULL), - QU( 8825177124486735972ULL), QU(13733447296837467838ULL), - QU( 6412293741681359625ULL), QU( 8313213138756135033ULL), - QU(11421481194803712517ULL), QU( 7997007691544174032ULL), - QU( 6812963847917605930ULL), QU( 9683091901227558641ULL), - QU(14703594165860324713ULL), QU( 1775476144519618309ULL), - QU( 2724283288516469519ULL), QU( 717642555185856868ULL), - QU( 8736402192215092346ULL), QU(11878800336431381021ULL), - QU( 4348816066017061293ULL), QU( 6115112756583631307ULL), - QU( 9176597239667142976ULL), QU(12615622714894259204ULL), - QU(10283406711301385987ULL), QU( 5111762509485379420ULL), - QU( 3118290051198688449ULL), QU( 7345123071632232145ULL), - QU( 9176423451688682359ULL), QU( 4843865456157868971ULL), - QU(12008036363752566088ULL), QU(12058837181919397720ULL), - QU( 2145073958457347366ULL), QU( 1526504881672818067ULL), - QU( 3488830105567134848ULL), QU(13208362960674805143ULL), - QU( 4077549672899572192ULL), QU( 7770995684693818365ULL), - QU( 1398532341546313593ULL), QU(12711859908703927840ULL), - QU( 1417561172594446813ULL), QU(17045191024194170604ULL), - QU( 4101933177604931713ULL), QU(14708428834203480320ULL), - QU(17447509264469407724ULL), QU(14314821973983434255ULL), - QU(17990472271061617265ULL), QU( 5087756685841673942ULL), - QU(12797820586893859939ULL), QU( 1778128952671092879ULL), - QU( 3535918530508665898ULL), QU( 9035729701042481301ULL), - QU(14808661568277079962ULL), QU(14587345077537747914ULL), - QU(11920080002323122708ULL), QU( 6426515805197278753ULL), - QU( 3295612216725984831ULL), QU(11040722532100876120ULL), - QU(12305952936387598754ULL), QU(16097391899742004253ULL), - QU( 4908537335606182208ULL), QU(12446674552196795504ULL), - QU(16010497855816895177ULL), QU( 9194378874788615551ULL), - QU( 3382957529567613384ULL), QU( 5154647600754974077ULL), - QU( 9801822865328396141ULL), QU( 9023662173919288143ULL), - QU(17623115353825147868ULL), QU( 8238115767443015816ULL), - QU(15811444159859002560ULL), QU( 9085612528904059661ULL), - QU( 6888601089398614254ULL), QU( 258252992894160189ULL), - QU( 6704363880792428622ULL), QU( 6114966032147235763ULL), - QU(11075393882690261875ULL), QU( 8797664238933620407ULL), - QU( 5901892006476726920ULL), QU( 5309780159285518958ULL), - QU(14940808387240817367ULL), QU(14642032021449656698ULL), - QU( 9808256672068504139ULL), QU( 3670135111380607658ULL), - QU(11211211097845960152ULL), QU( 1474304506716695808ULL), - QU(15843166204506876239ULL), QU( 7661051252471780561ULL), - QU(10170905502249418476ULL), QU( 7801416045582028589ULL), - QU( 2763981484737053050ULL), QU( 9491377905499253054ULL), - QU(16201395896336915095ULL), QU( 9256513756442782198ULL), - QU( 5411283157972456034ULL), QU( 5059433122288321676ULL), - QU( 4327408006721123357ULL), QU( 9278544078834433377ULL), - QU( 7601527110882281612ULL), QU(11848295896975505251ULL), - QU(12096998801094735560ULL), QU(14773480339823506413ULL), - QU(15586227433895802149ULL), QU(12786541257830242872ULL), - QU( 6904692985140503067ULL), QU( 5309011515263103959ULL), - QU(12105257191179371066ULL), QU(14654380212442225037ULL), - QU( 2556774974190695009ULL), QU( 4461297399927600261ULL), - QU(14888225660915118646ULL), QU(14915459341148291824ULL), - QU( 2738802166252327631ULL), QU( 6047155789239131512ULL), - QU(12920545353217010338ULL), QU(10697617257007840205ULL), - QU( 2751585253158203504ULL), QU(13252729159780047496ULL), - QU(14700326134672815469ULL), QU(14082527904374600529ULL), - QU(16852962273496542070ULL), QU(17446675504235853907ULL), - QU(15019600398527572311ULL), QU(12312781346344081551ULL), - QU(14524667935039810450ULL), QU( 5634005663377195738ULL), - QU(11375574739525000569ULL), QU( 2423665396433260040ULL), - QU( 5222836914796015410ULL), QU( 4397666386492647387ULL), - QU( 4619294441691707638ULL), QU( 665088602354770716ULL), - QU(13246495665281593610ULL), QU( 6564144270549729409ULL), - QU(10223216188145661688ULL), QU( 3961556907299230585ULL), - QU(11543262515492439914ULL), QU(16118031437285993790ULL), - QU( 7143417964520166465ULL), QU(13295053515909486772ULL), - QU( 40434666004899675ULL), QU(17127804194038347164ULL), - QU( 8599165966560586269ULL), QU( 8214016749011284903ULL), - QU(13725130352140465239ULL), QU( 5467254474431726291ULL), - QU( 7748584297438219877ULL), QU(16933551114829772472ULL), - QU( 2169618439506799400ULL), QU( 2169787627665113463ULL), - QU(17314493571267943764ULL), QU(18053575102911354912ULL), - QU(11928303275378476973ULL), QU(11593850925061715550ULL), - QU(17782269923473589362ULL), QU( 3280235307704747039ULL), - QU( 6145343578598685149ULL), QU(17080117031114086090ULL), - QU(18066839902983594755ULL), QU( 6517508430331020706ULL), - QU( 8092908893950411541ULL), QU(12558378233386153732ULL), - QU( 4476532167973132976ULL), QU(16081642430367025016ULL), - QU( 4233154094369139361ULL), QU( 8693630486693161027ULL), - QU(11244959343027742285ULL), QU(12273503967768513508ULL), - QU(14108978636385284876ULL), QU( 7242414665378826984ULL), - QU( 6561316938846562432ULL), QU( 8601038474994665795ULL), - QU(17532942353612365904ULL), QU(17940076637020912186ULL), - QU( 7340260368823171304ULL), QU( 7061807613916067905ULL), - QU(10561734935039519326ULL), QU(17990796503724650862ULL), - QU( 6208732943911827159ULL), QU( 359077562804090617ULL), - QU(14177751537784403113ULL), QU(10659599444915362902ULL), - QU(15081727220615085833ULL), QU(13417573895659757486ULL), - QU(15513842342017811524ULL), QU(11814141516204288231ULL), - QU( 1827312513875101814ULL), QU( 2804611699894603103ULL), - QU(17116500469975602763ULL), QU(12270191815211952087ULL), - QU(12256358467786024988ULL), QU(18435021722453971267ULL), - QU( 671330264390865618ULL), QU( 476504300460286050ULL), - QU(16465470901027093441ULL), QU( 4047724406247136402ULL), - QU( 1322305451411883346ULL), QU( 1388308688834322280ULL), - QU( 7303989085269758176ULL), QU( 9323792664765233642ULL), - QU( 4542762575316368936ULL), QU(17342696132794337618ULL), - QU( 4588025054768498379ULL), QU(13415475057390330804ULL), - QU(17880279491733405570ULL), QU(10610553400618620353ULL), - QU( 3180842072658960139ULL), QU(13002966655454270120ULL), - QU( 1665301181064982826ULL), QU( 7083673946791258979ULL), - QU( 190522247122496820ULL), QU(17388280237250677740ULL), - QU( 8430770379923642945ULL), QU(12987180971921668584ULL), - QU( 2311086108365390642ULL), QU( 2870984383579822345ULL), - QU(14014682609164653318ULL), QU(14467187293062251484ULL), - QU( 192186361147413298ULL), QU(15171951713531796524ULL), - QU( 9900305495015948728ULL), QU(17958004775615466344ULL), - QU(14346380954498606514ULL), QU(18040047357617407096ULL), - QU( 5035237584833424532ULL), QU(15089555460613972287ULL), - QU( 4131411873749729831ULL), QU( 1329013581168250330ULL), - QU(10095353333051193949ULL), QU(10749518561022462716ULL), - QU( 9050611429810755847ULL), QU(15022028840236655649ULL), - QU( 8775554279239748298ULL), QU(13105754025489230502ULL), - QU(15471300118574167585ULL), QU( 89864764002355628ULL), - QU( 8776416323420466637ULL), QU( 5280258630612040891ULL), - QU( 2719174488591862912ULL), QU( 7599309137399661994ULL), - QU(15012887256778039979ULL), QU(14062981725630928925ULL), - QU(12038536286991689603ULL), QU( 7089756544681775245ULL), - QU(10376661532744718039ULL), QU( 1265198725901533130ULL), - QU(13807996727081142408ULL), QU( 2935019626765036403ULL), - QU( 7651672460680700141ULL), QU( 3644093016200370795ULL), - QU( 2840982578090080674ULL), QU(17956262740157449201ULL), - QU(18267979450492880548ULL), QU(11799503659796848070ULL), - QU( 9942537025669672388ULL), QU(11886606816406990297ULL), - QU( 5488594946437447576ULL), QU( 7226714353282744302ULL), - QU( 3784851653123877043ULL), QU( 878018453244803041ULL), - QU(12110022586268616085ULL), QU( 734072179404675123ULL), - QU(11869573627998248542ULL), QU( 469150421297783998ULL), - QU( 260151124912803804ULL), QU(11639179410120968649ULL), - QU( 9318165193840846253ULL), QU(12795671722734758075ULL), - QU(15318410297267253933ULL), QU( 691524703570062620ULL), - QU( 5837129010576994601ULL), QU(15045963859726941052ULL), - QU( 5850056944932238169ULL), QU(12017434144750943807ULL), - QU( 7447139064928956574ULL), QU( 3101711812658245019ULL), - QU(16052940704474982954ULL), QU(18195745945986994042ULL), - QU( 8932252132785575659ULL), QU(13390817488106794834ULL), - QU(11582771836502517453ULL), QU( 4964411326683611686ULL), - QU( 2195093981702694011ULL), QU(14145229538389675669ULL), - QU(16459605532062271798ULL), QU( 866316924816482864ULL), - QU( 4593041209937286377ULL), QU( 8415491391910972138ULL), - QU( 4171236715600528969ULL), QU(16637569303336782889ULL), - QU( 2002011073439212680ULL), QU(17695124661097601411ULL), - QU( 4627687053598611702ULL), QU( 7895831936020190403ULL), - QU( 8455951300917267802ULL), QU( 2923861649108534854ULL), - QU( 8344557563927786255ULL), QU( 6408671940373352556ULL), - QU(12210227354536675772ULL), QU(14294804157294222295ULL), - QU(10103022425071085127ULL), QU(10092959489504123771ULL), - QU( 6554774405376736268ULL), QU(12629917718410641774ULL), - QU( 6260933257596067126ULL), QU( 2460827021439369673ULL), - QU( 2541962996717103668ULL), QU( 597377203127351475ULL), - QU( 5316984203117315309ULL), QU( 4811211393563241961ULL), - QU(13119698597255811641ULL), QU( 8048691512862388981ULL), - QU(10216818971194073842ULL), QU( 4612229970165291764ULL), - QU(10000980798419974770ULL), QU( 6877640812402540687ULL), - QU( 1488727563290436992ULL), QU( 2227774069895697318ULL), - QU(11237754507523316593ULL), QU(13478948605382290972ULL), - QU( 1963583846976858124ULL), QU( 5512309205269276457ULL), - QU( 3972770164717652347ULL), QU( 3841751276198975037ULL), - QU(10283343042181903117ULL), QU( 8564001259792872199ULL), - QU(16472187244722489221ULL), QU( 8953493499268945921ULL), - QU( 3518747340357279580ULL), QU( 4003157546223963073ULL), - QU( 3270305958289814590ULL), QU( 3966704458129482496ULL), - QU( 8122141865926661939ULL), QU(14627734748099506653ULL), - QU(13064426990862560568ULL), QU( 2414079187889870829ULL), - QU( 5378461209354225306ULL), QU(10841985740128255566ULL), - QU( 538582442885401738ULL), QU( 7535089183482905946ULL), - QU(16117559957598879095ULL), QU( 8477890721414539741ULL), - QU( 1459127491209533386ULL), QU(17035126360733620462ULL), - QU( 8517668552872379126ULL), QU(10292151468337355014ULL), - QU(17081267732745344157ULL), QU(13751455337946087178ULL), - QU(14026945459523832966ULL), QU( 6653278775061723516ULL), - QU(10619085543856390441ULL), QU( 2196343631481122885ULL), - QU(10045966074702826136ULL), QU(10082317330452718282ULL), - QU( 5920859259504831242ULL), QU( 9951879073426540617ULL), - QU( 7074696649151414158ULL), QU(15808193543879464318ULL), - QU( 7385247772746953374ULL), QU( 3192003544283864292ULL), - QU(18153684490917593847ULL), QU(12423498260668568905ULL), - QU(10957758099756378169ULL), QU(11488762179911016040ULL), - QU( 2099931186465333782ULL), QU(11180979581250294432ULL), - QU( 8098916250668367933ULL), QU( 3529200436790763465ULL), - QU(12988418908674681745ULL), QU( 6147567275954808580ULL), - QU( 3207503344604030989ULL), QU(10761592604898615360ULL), - QU( 229854861031893504ULL), QU( 8809853962667144291ULL), - QU(13957364469005693860ULL), QU( 7634287665224495886ULL), - QU(12353487366976556874ULL), QU( 1134423796317152034ULL), - QU( 2088992471334107068ULL), QU( 7393372127190799698ULL), - QU( 1845367839871058391ULL), QU( 207922563987322884ULL), - QU(11960870813159944976ULL), QU(12182120053317317363ULL), - QU(17307358132571709283ULL), QU(13871081155552824936ULL), - QU(18304446751741566262ULL), QU( 7178705220184302849ULL), - QU(10929605677758824425ULL), QU(16446976977835806844ULL), - QU(13723874412159769044ULL), QU( 6942854352100915216ULL), - QU( 1726308474365729390ULL), QU( 2150078766445323155ULL), - QU(15345558947919656626ULL), QU(12145453828874527201ULL), - QU( 2054448620739726849ULL), QU( 2740102003352628137ULL), - QU(11294462163577610655ULL), QU( 756164283387413743ULL), - QU(17841144758438810880ULL), QU(10802406021185415861ULL), - QU( 8716455530476737846ULL), QU( 6321788834517649606ULL), - QU(14681322910577468426ULL), QU(17330043563884336387ULL), - QU(12701802180050071614ULL), QU(14695105111079727151ULL), - QU( 5112098511654172830ULL), QU( 4957505496794139973ULL), - QU( 8270979451952045982ULL), QU(12307685939199120969ULL), - QU(12425799408953443032ULL), QU( 8376410143634796588ULL), - QU(16621778679680060464ULL), QU( 3580497854566660073ULL), - QU( 1122515747803382416ULL), QU( 857664980960597599ULL), - QU( 6343640119895925918ULL), QU(12878473260854462891ULL), - QU(10036813920765722626ULL), QU(14451335468363173812ULL), - QU( 5476809692401102807ULL), QU(16442255173514366342ULL), - QU(13060203194757167104ULL), QU(14354124071243177715ULL), - QU(15961249405696125227ULL), QU(13703893649690872584ULL), - QU( 363907326340340064ULL), QU( 6247455540491754842ULL), - QU(12242249332757832361ULL), QU( 156065475679796717ULL), - QU( 9351116235749732355ULL), QU( 4590350628677701405ULL), - QU( 1671195940982350389ULL), QU(13501398458898451905ULL), - QU( 6526341991225002255ULL), QU( 1689782913778157592ULL), - QU( 7439222350869010334ULL), QU(13975150263226478308ULL), - QU(11411961169932682710ULL), QU(17204271834833847277ULL), - QU( 541534742544435367ULL), QU( 6591191931218949684ULL), - QU( 2645454775478232486ULL), QU( 4322857481256485321ULL), - QU( 8477416487553065110ULL), QU(12902505428548435048ULL), - QU( 971445777981341415ULL), QU(14995104682744976712ULL), - QU( 4243341648807158063ULL), QU( 8695061252721927661ULL), - QU( 5028202003270177222ULL), QU( 2289257340915567840ULL), - QU(13870416345121866007ULL), QU(13994481698072092233ULL), - QU( 6912785400753196481ULL), QU( 2278309315841980139ULL), - QU( 4329765449648304839ULL), QU( 5963108095785485298ULL), - QU( 4880024847478722478ULL), QU(16015608779890240947ULL), - QU( 1866679034261393544ULL), QU( 914821179919731519ULL), - QU( 9643404035648760131ULL), QU( 2418114953615593915ULL), - QU( 944756836073702374ULL), QU(15186388048737296834ULL), - QU( 7723355336128442206ULL), QU( 7500747479679599691ULL), - QU(18013961306453293634ULL), QU( 2315274808095756456ULL), - QU(13655308255424029566ULL), QU(17203800273561677098ULL), - QU( 1382158694422087756ULL), QU( 5090390250309588976ULL), - QU( 517170818384213989ULL), QU( 1612709252627729621ULL), - QU( 1330118955572449606ULL), QU( 300922478056709885ULL), - QU(18115693291289091987ULL), QU(13491407109725238321ULL), - QU(15293714633593827320ULL), QU( 5151539373053314504ULL), - QU( 5951523243743139207ULL), QU(14459112015249527975ULL), - QU( 5456113959000700739ULL), QU( 3877918438464873016ULL), - QU(12534071654260163555ULL), QU(15871678376893555041ULL), - QU(11005484805712025549ULL), QU(16353066973143374252ULL), - QU( 4358331472063256685ULL), QU( 8268349332210859288ULL), - QU(12485161590939658075ULL), QU(13955993592854471343ULL), - QU( 5911446886848367039ULL), QU(14925834086813706974ULL), - QU( 6590362597857994805ULL), QU( 1280544923533661875ULL), - QU( 1637756018947988164ULL), QU( 4734090064512686329ULL), - QU(16693705263131485912ULL), QU( 6834882340494360958ULL), - QU( 8120732176159658505ULL), QU( 2244371958905329346ULL), - QU(10447499707729734021ULL), QU( 7318742361446942194ULL), - QU( 8032857516355555296ULL), QU(14023605983059313116ULL), - QU( 1032336061815461376ULL), QU( 9840995337876562612ULL), - QU( 9869256223029203587ULL), QU(12227975697177267636ULL), - QU(12728115115844186033ULL), QU( 7752058479783205470ULL), - QU( 729733219713393087ULL), QU(12954017801239007622ULL) + KQU(16924766246869039260), KQU( 8201438687333352714), + KQU( 2265290287015001750), KQU(18397264611805473832), + KQU( 3375255223302384358), KQU( 6345559975416828796), + KQU(18229739242790328073), KQU( 7596792742098800905), + KQU( 255338647169685981), KQU( 2052747240048610300), + KQU(18328151576097299343), KQU(12472905421133796567), + KQU(11315245349717600863), KQU(16594110197775871209), + KQU(15708751964632456450), KQU(10452031272054632535), + KQU(11097646720811454386), KQU( 4556090668445745441), + KQU(17116187693090663106), KQU(14931526836144510645), + KQU( 9190752218020552591), KQU( 9625800285771901401), + KQU(13995141077659972832), KQU( 5194209094927829625), + KQU( 4156788379151063303), KQU( 8523452593770139494), + KQU(14082382103049296727), KQU( 2462601863986088483), + KQU( 3030583461592840678), KQU( 5221622077872827681), + KQU( 3084210671228981236), KQU(13956758381389953823), + KQU(13503889856213423831), KQU(15696904024189836170), + KQU( 4612584152877036206), KQU( 6231135538447867881), + KQU(10172457294158869468), KQU( 6452258628466708150), + KQU(14044432824917330221), KQU( 370168364480044279), + KQU(10102144686427193359), KQU( 667870489994776076), + KQU( 2732271956925885858), KQU(18027788905977284151), + KQU(15009842788582923859), KQU( 7136357960180199542), + KQU(15901736243475578127), KQU(16951293785352615701), + KQU(10551492125243691632), KQU(17668869969146434804), + KQU(13646002971174390445), KQU( 9804471050759613248), + KQU( 5511670439655935493), KQU(18103342091070400926), + KQU(17224512747665137533), KQU(15534627482992618168), + KQU( 1423813266186582647), KQU(15821176807932930024), + KQU( 30323369733607156), KQU(11599382494723479403), + KQU( 653856076586810062), KQU( 3176437395144899659), + KQU(14028076268147963917), KQU(16156398271809666195), + KQU( 3166955484848201676), KQU( 5746805620136919390), + KQU(17297845208891256593), KQU(11691653183226428483), + KQU(17900026146506981577), KQU(15387382115755971042), + KQU(16923567681040845943), KQU( 8039057517199388606), + KQU(11748409241468629263), KQU( 794358245539076095), + KQU(13438501964693401242), KQU(14036803236515618962), + KQU( 5252311215205424721), KQU(17806589612915509081), + KQU( 6802767092397596006), KQU(14212120431184557140), + KQU( 1072951366761385712), KQU(13098491780722836296), + KQU( 9466676828710797353), KQU(12673056849042830081), + KQU(12763726623645357580), KQU(16468961652999309493), + KQU(15305979875636438926), KQU(17444713151223449734), + KQU( 5692214267627883674), KQU(13049589139196151505), + KQU( 880115207831670745), KQU( 1776529075789695498), + KQU(16695225897801466485), KQU(10666901778795346845), + KQU( 6164389346722833869), KQU( 2863817793264300475), + KQU( 9464049921886304754), KQU( 3993566636740015468), + KQU( 9983749692528514136), KQU(16375286075057755211), + KQU(16042643417005440820), KQU(11445419662923489877), + KQU( 7999038846885158836), KQU( 6721913661721511535), + KQU( 5363052654139357320), KQU( 1817788761173584205), + KQU(13290974386445856444), KQU( 4650350818937984680), + KQU( 8219183528102484836), KQU( 1569862923500819899), + KQU( 4189359732136641860), KQU(14202822961683148583), + KQU( 4457498315309429058), KQU(13089067387019074834), + KQU(11075517153328927293), KQU(10277016248336668389), + KQU( 7070509725324401122), KQU(17808892017780289380), + KQU(13143367339909287349), KQU( 1377743745360085151), + KQU( 5749341807421286485), KQU(14832814616770931325), + KQU( 7688820635324359492), KQU(10960474011539770045), + KQU( 81970066653179790), KQU(12619476072607878022), + KQU( 4419566616271201744), KQU(15147917311750568503), + KQU( 5549739182852706345), KQU( 7308198397975204770), + KQU(13580425496671289278), KQU(17070764785210130301), + KQU( 8202832846285604405), KQU( 6873046287640887249), + KQU( 6927424434308206114), KQU( 6139014645937224874), + KQU(10290373645978487639), KQU(15904261291701523804), + KQU( 9628743442057826883), KQU(18383429096255546714), + KQU( 4977413265753686967), KQU( 7714317492425012869), + KQU( 9025232586309926193), KQU(14627338359776709107), + KQU(14759849896467790763), KQU(10931129435864423252), + KQU( 4588456988775014359), KQU(10699388531797056724), + KQU( 468652268869238792), KQU( 5755943035328078086), + KQU( 2102437379988580216), KQU( 9986312786506674028), + KQU( 2654207180040945604), KQU( 8726634790559960062), + KQU( 100497234871808137), KQU( 2800137176951425819), + KQU( 6076627612918553487), KQU( 5780186919186152796), + KQU( 8179183595769929098), KQU( 6009426283716221169), + KQU( 2796662551397449358), KQU( 1756961367041986764), + KQU( 6972897917355606205), KQU(14524774345368968243), + KQU( 2773529684745706940), KQU( 4853632376213075959), + KQU( 4198177923731358102), KQU( 8271224913084139776), + KQU( 2741753121611092226), KQU(16782366145996731181), + KQU(15426125238972640790), KQU(13595497100671260342), + KQU( 3173531022836259898), KQU( 6573264560319511662), + KQU(18041111951511157441), KQU( 2351433581833135952), + KQU( 3113255578908173487), KQU( 1739371330877858784), + KQU(16046126562789165480), KQU( 8072101652214192925), + KQU(15267091584090664910), KQU( 9309579200403648940), + KQU( 5218892439752408722), KQU(14492477246004337115), + KQU(17431037586679770619), KQU( 7385248135963250480), + KQU( 9580144956565560660), KQU( 4919546228040008720), + KQU(15261542469145035584), KQU(18233297270822253102), + KQU( 5453248417992302857), KQU( 9309519155931460285), + KQU(10342813012345291756), KQU(15676085186784762381), + KQU(15912092950691300645), KQU( 9371053121499003195), + KQU( 9897186478226866746), KQU(14061858287188196327), + KQU( 122575971620788119), KQU(12146750969116317754), + KQU( 4438317272813245201), KQU( 8332576791009527119), + KQU(13907785691786542057), KQU(10374194887283287467), + KQU( 2098798755649059566), KQU( 3416235197748288894), + KQU( 8688269957320773484), KQU( 7503964602397371571), + KQU(16724977015147478236), KQU( 9461512855439858184), + KQU(13259049744534534727), KQU( 3583094952542899294), + KQU( 8764245731305528292), KQU(13240823595462088985), + KQU(13716141617617910448), KQU(18114969519935960955), + KQU( 2297553615798302206), KQU( 4585521442944663362), + KQU(17776858680630198686), KQU( 4685873229192163363), + KQU( 152558080671135627), KQU(15424900540842670088), + KQU(13229630297130024108), KQU(17530268788245718717), + KQU(16675633913065714144), KQU( 3158912717897568068), + KQU(15399132185380087288), KQU( 7401418744515677872), + KQU(13135412922344398535), KQU( 6385314346100509511), + KQU(13962867001134161139), KQU(10272780155442671999), + KQU(12894856086597769142), KQU(13340877795287554994), + KQU(12913630602094607396), KQU(12543167911119793857), + KQU(17343570372251873096), KQU(10959487764494150545), + KQU( 6966737953093821128), KQU(13780699135496988601), + KQU( 4405070719380142046), KQU(14923788365607284982), + KQU( 2869487678905148380), KQU( 6416272754197188403), + KQU(15017380475943612591), KQU( 1995636220918429487), + KQU( 3402016804620122716), KQU(15800188663407057080), + KQU(11362369990390932882), KQU(15262183501637986147), + KQU(10239175385387371494), KQU( 9352042420365748334), + KQU( 1682457034285119875), KQU( 1724710651376289644), + KQU( 2038157098893817966), KQU( 9897825558324608773), + KQU( 1477666236519164736), KQU(16835397314511233640), + KQU(10370866327005346508), KQU(10157504370660621982), + KQU(12113904045335882069), KQU(13326444439742783008), + KQU(11302769043000765804), KQU(13594979923955228484), + KQU(11779351762613475968), KQU( 3786101619539298383), + KQU( 8021122969180846063), KQU(15745904401162500495), + KQU(10762168465993897267), KQU(13552058957896319026), + KQU(11200228655252462013), KQU( 5035370357337441226), + KQU( 7593918984545500013), KQU( 5418554918361528700), + KQU( 4858270799405446371), KQU( 9974659566876282544), + KQU(18227595922273957859), KQU( 2772778443635656220), + KQU(14285143053182085385), KQU( 9939700992429600469), + KQU(12756185904545598068), KQU( 2020783375367345262), + KQU( 57026775058331227), KQU( 950827867930065454), + KQU( 6602279670145371217), KQU( 2291171535443566929), + KQU( 5832380724425010313), KQU( 1220343904715982285), + KQU(17045542598598037633), KQU(15460481779702820971), + KQU(13948388779949365130), KQU(13975040175430829518), + KQU(17477538238425541763), KQU(11104663041851745725), + KQU(15860992957141157587), KQU(14529434633012950138), + KQU( 2504838019075394203), KQU( 7512113882611121886), + KQU( 4859973559980886617), KQU( 1258601555703250219), + KQU(15594548157514316394), KQU( 4516730171963773048), + KQU(11380103193905031983), KQU( 6809282239982353344), + KQU(18045256930420065002), KQU( 2453702683108791859), + KQU( 977214582986981460), KQU( 2006410402232713466), + KQU( 6192236267216378358), KQU( 3429468402195675253), + KQU(18146933153017348921), KQU(17369978576367231139), + KQU( 1246940717230386603), KQU(11335758870083327110), + KQU(14166488801730353682), KQU( 9008573127269635732), + KQU(10776025389820643815), KQU(15087605441903942962), + KQU( 1359542462712147922), KQU(13898874411226454206), + KQU(17911176066536804411), KQU( 9435590428600085274), + KQU( 294488509967864007), KQU( 8890111397567922046), + KQU( 7987823476034328778), KQU(13263827582440967651), + KQU( 7503774813106751573), KQU(14974747296185646837), + KQU( 8504765037032103375), KQU(17340303357444536213), + KQU( 7704610912964485743), KQU( 8107533670327205061), + KQU( 9062969835083315985), KQU(16968963142126734184), + KQU(12958041214190810180), KQU( 2720170147759570200), + KQU( 2986358963942189566), KQU(14884226322219356580), + KQU( 286224325144368520), KQU(11313800433154279797), + KQU(18366849528439673248), KQU(17899725929482368789), + KQU( 3730004284609106799), KQU( 1654474302052767205), + KQU( 5006698007047077032), KQU( 8196893913601182838), + KQU(15214541774425211640), KQU(17391346045606626073), + KQU( 8369003584076969089), KQU( 3939046733368550293), + KQU(10178639720308707785), KQU( 2180248669304388697), + KQU( 62894391300126322), KQU( 9205708961736223191), + KQU( 6837431058165360438), KQU( 3150743890848308214), + KQU(17849330658111464583), KQU(12214815643135450865), + KQU(13410713840519603402), KQU( 3200778126692046802), + KQU(13354780043041779313), KQU( 800850022756886036), + KQU(15660052933953067433), KQU( 6572823544154375676), + KQU(11030281857015819266), KQU(12682241941471433835), + KQU(11654136407300274693), KQU( 4517795492388641109), + KQU( 9757017371504524244), KQU(17833043400781889277), + KQU(12685085201747792227), KQU(10408057728835019573), + KQU( 98370418513455221), KQU( 6732663555696848598), + KQU(13248530959948529780), KQU( 3530441401230622826), + KQU(18188251992895660615), KQU( 1847918354186383756), + KQU( 1127392190402660921), KQU(11293734643143819463), + KQU( 3015506344578682982), KQU(13852645444071153329), + KQU( 2121359659091349142), KQU( 1294604376116677694), + KQU( 5616576231286352318), KQU( 7112502442954235625), + KQU(11676228199551561689), KQU(12925182803007305359), + KQU( 7852375518160493082), KQU( 1136513130539296154), + KQU( 5636923900916593195), KQU( 3221077517612607747), + KQU(17784790465798152513), KQU( 3554210049056995938), + KQU(17476839685878225874), KQU( 3206836372585575732), + KQU( 2765333945644823430), KQU(10080070903718799528), + KQU( 5412370818878286353), KQU( 9689685887726257728), + KQU( 8236117509123533998), KQU( 1951139137165040214), + KQU( 4492205209227980349), KQU(16541291230861602967), + KQU( 1424371548301437940), KQU( 9117562079669206794), + KQU(14374681563251691625), KQU(13873164030199921303), + KQU( 6680317946770936731), KQU(15586334026918276214), + KQU(10896213950976109802), KQU( 9506261949596413689), + KQU( 9903949574308040616), KQU( 6038397344557204470), + KQU( 174601465422373648), KQU(15946141191338238030), + KQU(17142225620992044937), KQU( 7552030283784477064), + KQU( 2947372384532947997), KQU( 510797021688197711), + KQU( 4962499439249363461), KQU( 23770320158385357), + KQU( 959774499105138124), KQU( 1468396011518788276), + KQU( 2015698006852312308), KQU( 4149400718489980136), + KQU( 5992916099522371188), KQU(10819182935265531076), + KQU(16189787999192351131), KQU( 342833961790261950), + KQU(12470830319550495336), KQU(18128495041912812501), + KQU( 1193600899723524337), KQU( 9056793666590079770), + KQU( 2154021227041669041), KQU( 4963570213951235735), + KQU( 4865075960209211409), KQU( 2097724599039942963), + KQU( 2024080278583179845), KQU(11527054549196576736), + KQU(10650256084182390252), KQU( 4808408648695766755), + KQU( 1642839215013788844), KQU(10607187948250398390), + KQU( 7076868166085913508), KQU( 730522571106887032), + KQU(12500579240208524895), KQU( 4484390097311355324), + KQU(15145801330700623870), KQU( 8055827661392944028), + KQU( 5865092976832712268), KQU(15159212508053625143), + KQU( 3560964582876483341), KQU( 4070052741344438280), + KQU( 6032585709886855634), KQU(15643262320904604873), + KQU( 2565119772293371111), KQU( 318314293065348260), + KQU(15047458749141511872), KQU( 7772788389811528730), + KQU( 7081187494343801976), KQU( 6465136009467253947), + KQU(10425940692543362069), KQU( 554608190318339115), + KQU(14796699860302125214), KQU( 1638153134431111443), + KQU(10336967447052276248), KQU( 8412308070396592958), + KQU( 4004557277152051226), KQU( 8143598997278774834), + KQU(16413323996508783221), KQU(13139418758033994949), + KQU( 9772709138335006667), KQU( 2818167159287157659), + KQU(17091740573832523669), KQU(14629199013130751608), + KQU(18268322711500338185), KQU( 8290963415675493063), + KQU( 8830864907452542588), KQU( 1614839084637494849), + KQU(14855358500870422231), KQU( 3472996748392519937), + KQU(15317151166268877716), KQU( 5825895018698400362), + KQU(16730208429367544129), KQU(10481156578141202800), + KQU( 4746166512382823750), KQU(12720876014472464998), + KQU( 8825177124486735972), KQU(13733447296837467838), + KQU( 6412293741681359625), KQU( 8313213138756135033), + KQU(11421481194803712517), KQU( 7997007691544174032), + KQU( 6812963847917605930), KQU( 9683091901227558641), + KQU(14703594165860324713), KQU( 1775476144519618309), + KQU( 2724283288516469519), KQU( 717642555185856868), + KQU( 8736402192215092346), KQU(11878800336431381021), + KQU( 4348816066017061293), KQU( 6115112756583631307), + KQU( 9176597239667142976), KQU(12615622714894259204), + KQU(10283406711301385987), KQU( 5111762509485379420), + KQU( 3118290051198688449), KQU( 7345123071632232145), + KQU( 9176423451688682359), KQU( 4843865456157868971), + KQU(12008036363752566088), KQU(12058837181919397720), + KQU( 2145073958457347366), KQU( 1526504881672818067), + KQU( 3488830105567134848), KQU(13208362960674805143), + KQU( 4077549672899572192), KQU( 7770995684693818365), + KQU( 1398532341546313593), KQU(12711859908703927840), + KQU( 1417561172594446813), KQU(17045191024194170604), + KQU( 4101933177604931713), KQU(14708428834203480320), + KQU(17447509264469407724), KQU(14314821973983434255), + KQU(17990472271061617265), KQU( 5087756685841673942), + KQU(12797820586893859939), KQU( 1778128952671092879), + KQU( 3535918530508665898), KQU( 9035729701042481301), + KQU(14808661568277079962), KQU(14587345077537747914), + KQU(11920080002323122708), KQU( 6426515805197278753), + KQU( 3295612216725984831), KQU(11040722532100876120), + KQU(12305952936387598754), KQU(16097391899742004253), + KQU( 4908537335606182208), KQU(12446674552196795504), + KQU(16010497855816895177), KQU( 9194378874788615551), + KQU( 3382957529567613384), KQU( 5154647600754974077), + KQU( 9801822865328396141), KQU( 9023662173919288143), + KQU(17623115353825147868), KQU( 8238115767443015816), + KQU(15811444159859002560), KQU( 9085612528904059661), + KQU( 6888601089398614254), KQU( 258252992894160189), + KQU( 6704363880792428622), KQU( 6114966032147235763), + KQU(11075393882690261875), KQU( 8797664238933620407), + KQU( 5901892006476726920), KQU( 5309780159285518958), + KQU(14940808387240817367), KQU(14642032021449656698), + KQU( 9808256672068504139), KQU( 3670135111380607658), + KQU(11211211097845960152), KQU( 1474304506716695808), + KQU(15843166204506876239), KQU( 7661051252471780561), + KQU(10170905502249418476), KQU( 7801416045582028589), + KQU( 2763981484737053050), KQU( 9491377905499253054), + KQU(16201395896336915095), KQU( 9256513756442782198), + KQU( 5411283157972456034), KQU( 5059433122288321676), + KQU( 4327408006721123357), KQU( 9278544078834433377), + KQU( 7601527110882281612), KQU(11848295896975505251), + KQU(12096998801094735560), KQU(14773480339823506413), + KQU(15586227433895802149), KQU(12786541257830242872), + KQU( 6904692985140503067), KQU( 5309011515263103959), + KQU(12105257191179371066), KQU(14654380212442225037), + KQU( 2556774974190695009), KQU( 4461297399927600261), + KQU(14888225660915118646), KQU(14915459341148291824), + KQU( 2738802166252327631), KQU( 6047155789239131512), + KQU(12920545353217010338), KQU(10697617257007840205), + KQU( 2751585253158203504), KQU(13252729159780047496), + KQU(14700326134672815469), KQU(14082527904374600529), + KQU(16852962273496542070), KQU(17446675504235853907), + KQU(15019600398527572311), KQU(12312781346344081551), + KQU(14524667935039810450), KQU( 5634005663377195738), + KQU(11375574739525000569), KQU( 2423665396433260040), + KQU( 5222836914796015410), KQU( 4397666386492647387), + KQU( 4619294441691707638), KQU( 665088602354770716), + KQU(13246495665281593610), KQU( 6564144270549729409), + KQU(10223216188145661688), KQU( 3961556907299230585), + KQU(11543262515492439914), KQU(16118031437285993790), + KQU( 7143417964520166465), KQU(13295053515909486772), + KQU( 40434666004899675), KQU(17127804194038347164), + KQU( 8599165966560586269), KQU( 8214016749011284903), + KQU(13725130352140465239), KQU( 5467254474431726291), + KQU( 7748584297438219877), KQU(16933551114829772472), + KQU( 2169618439506799400), KQU( 2169787627665113463), + KQU(17314493571267943764), KQU(18053575102911354912), + KQU(11928303275378476973), KQU(11593850925061715550), + KQU(17782269923473589362), KQU( 3280235307704747039), + KQU( 6145343578598685149), KQU(17080117031114086090), + KQU(18066839902983594755), KQU( 6517508430331020706), + KQU( 8092908893950411541), KQU(12558378233386153732), + KQU( 4476532167973132976), KQU(16081642430367025016), + KQU( 4233154094369139361), KQU( 8693630486693161027), + KQU(11244959343027742285), KQU(12273503967768513508), + KQU(14108978636385284876), KQU( 7242414665378826984), + KQU( 6561316938846562432), KQU( 8601038474994665795), + KQU(17532942353612365904), KQU(17940076637020912186), + KQU( 7340260368823171304), KQU( 7061807613916067905), + KQU(10561734935039519326), KQU(17990796503724650862), + KQU( 6208732943911827159), KQU( 359077562804090617), + KQU(14177751537784403113), KQU(10659599444915362902), + KQU(15081727220615085833), KQU(13417573895659757486), + KQU(15513842342017811524), KQU(11814141516204288231), + KQU( 1827312513875101814), KQU( 2804611699894603103), + KQU(17116500469975602763), KQU(12270191815211952087), + KQU(12256358467786024988), KQU(18435021722453971267), + KQU( 671330264390865618), KQU( 476504300460286050), + KQU(16465470901027093441), KQU( 4047724406247136402), + KQU( 1322305451411883346), KQU( 1388308688834322280), + KQU( 7303989085269758176), KQU( 9323792664765233642), + KQU( 4542762575316368936), KQU(17342696132794337618), + KQU( 4588025054768498379), KQU(13415475057390330804), + KQU(17880279491733405570), KQU(10610553400618620353), + KQU( 3180842072658960139), KQU(13002966655454270120), + KQU( 1665301181064982826), KQU( 7083673946791258979), + KQU( 190522247122496820), KQU(17388280237250677740), + KQU( 8430770379923642945), KQU(12987180971921668584), + KQU( 2311086108365390642), KQU( 2870984383579822345), + KQU(14014682609164653318), KQU(14467187293062251484), + KQU( 192186361147413298), KQU(15171951713531796524), + KQU( 9900305495015948728), KQU(17958004775615466344), + KQU(14346380954498606514), KQU(18040047357617407096), + KQU( 5035237584833424532), KQU(15089555460613972287), + KQU( 4131411873749729831), KQU( 1329013581168250330), + KQU(10095353333051193949), KQU(10749518561022462716), + KQU( 9050611429810755847), KQU(15022028840236655649), + KQU( 8775554279239748298), KQU(13105754025489230502), + KQU(15471300118574167585), KQU( 89864764002355628), + KQU( 8776416323420466637), KQU( 5280258630612040891), + KQU( 2719174488591862912), KQU( 7599309137399661994), + KQU(15012887256778039979), KQU(14062981725630928925), + KQU(12038536286991689603), KQU( 7089756544681775245), + KQU(10376661532744718039), KQU( 1265198725901533130), + KQU(13807996727081142408), KQU( 2935019626765036403), + KQU( 7651672460680700141), KQU( 3644093016200370795), + KQU( 2840982578090080674), KQU(17956262740157449201), + KQU(18267979450492880548), KQU(11799503659796848070), + KQU( 9942537025669672388), KQU(11886606816406990297), + KQU( 5488594946437447576), KQU( 7226714353282744302), + KQU( 3784851653123877043), KQU( 878018453244803041), + KQU(12110022586268616085), KQU( 734072179404675123), + KQU(11869573627998248542), KQU( 469150421297783998), + KQU( 260151124912803804), KQU(11639179410120968649), + KQU( 9318165193840846253), KQU(12795671722734758075), + KQU(15318410297267253933), KQU( 691524703570062620), + KQU( 5837129010576994601), KQU(15045963859726941052), + KQU( 5850056944932238169), KQU(12017434144750943807), + KQU( 7447139064928956574), KQU( 3101711812658245019), + KQU(16052940704474982954), KQU(18195745945986994042), + KQU( 8932252132785575659), KQU(13390817488106794834), + KQU(11582771836502517453), KQU( 4964411326683611686), + KQU( 2195093981702694011), KQU(14145229538389675669), + KQU(16459605532062271798), KQU( 866316924816482864), + KQU( 4593041209937286377), KQU( 8415491391910972138), + KQU( 4171236715600528969), KQU(16637569303336782889), + KQU( 2002011073439212680), KQU(17695124661097601411), + KQU( 4627687053598611702), KQU( 7895831936020190403), + KQU( 8455951300917267802), KQU( 2923861649108534854), + KQU( 8344557563927786255), KQU( 6408671940373352556), + KQU(12210227354536675772), KQU(14294804157294222295), + KQU(10103022425071085127), KQU(10092959489504123771), + KQU( 6554774405376736268), KQU(12629917718410641774), + KQU( 6260933257596067126), KQU( 2460827021439369673), + KQU( 2541962996717103668), KQU( 597377203127351475), + KQU( 5316984203117315309), KQU( 4811211393563241961), + KQU(13119698597255811641), KQU( 8048691512862388981), + KQU(10216818971194073842), KQU( 4612229970165291764), + KQU(10000980798419974770), KQU( 6877640812402540687), + KQU( 1488727563290436992), KQU( 2227774069895697318), + KQU(11237754507523316593), KQU(13478948605382290972), + KQU( 1963583846976858124), KQU( 5512309205269276457), + KQU( 3972770164717652347), KQU( 3841751276198975037), + KQU(10283343042181903117), KQU( 8564001259792872199), + KQU(16472187244722489221), KQU( 8953493499268945921), + KQU( 3518747340357279580), KQU( 4003157546223963073), + KQU( 3270305958289814590), KQU( 3966704458129482496), + KQU( 8122141865926661939), KQU(14627734748099506653), + KQU(13064426990862560568), KQU( 2414079187889870829), + KQU( 5378461209354225306), KQU(10841985740128255566), + KQU( 538582442885401738), KQU( 7535089183482905946), + KQU(16117559957598879095), KQU( 8477890721414539741), + KQU( 1459127491209533386), KQU(17035126360733620462), + KQU( 8517668552872379126), KQU(10292151468337355014), + KQU(17081267732745344157), KQU(13751455337946087178), + KQU(14026945459523832966), KQU( 6653278775061723516), + KQU(10619085543856390441), KQU( 2196343631481122885), + KQU(10045966074702826136), KQU(10082317330452718282), + KQU( 5920859259504831242), KQU( 9951879073426540617), + KQU( 7074696649151414158), KQU(15808193543879464318), + KQU( 7385247772746953374), KQU( 3192003544283864292), + KQU(18153684490917593847), KQU(12423498260668568905), + KQU(10957758099756378169), KQU(11488762179911016040), + KQU( 2099931186465333782), KQU(11180979581250294432), + KQU( 8098916250668367933), KQU( 3529200436790763465), + KQU(12988418908674681745), KQU( 6147567275954808580), + KQU( 3207503344604030989), KQU(10761592604898615360), + KQU( 229854861031893504), KQU( 8809853962667144291), + KQU(13957364469005693860), KQU( 7634287665224495886), + KQU(12353487366976556874), KQU( 1134423796317152034), + KQU( 2088992471334107068), KQU( 7393372127190799698), + KQU( 1845367839871058391), KQU( 207922563987322884), + KQU(11960870813159944976), KQU(12182120053317317363), + KQU(17307358132571709283), KQU(13871081155552824936), + KQU(18304446751741566262), KQU( 7178705220184302849), + KQU(10929605677758824425), KQU(16446976977835806844), + KQU(13723874412159769044), KQU( 6942854352100915216), + KQU( 1726308474365729390), KQU( 2150078766445323155), + KQU(15345558947919656626), KQU(12145453828874527201), + KQU( 2054448620739726849), KQU( 2740102003352628137), + KQU(11294462163577610655), KQU( 756164283387413743), + KQU(17841144758438810880), KQU(10802406021185415861), + KQU( 8716455530476737846), KQU( 6321788834517649606), + KQU(14681322910577468426), KQU(17330043563884336387), + KQU(12701802180050071614), KQU(14695105111079727151), + KQU( 5112098511654172830), KQU( 4957505496794139973), + KQU( 8270979451952045982), KQU(12307685939199120969), + KQU(12425799408953443032), KQU( 8376410143634796588), + KQU(16621778679680060464), KQU( 3580497854566660073), + KQU( 1122515747803382416), KQU( 857664980960597599), + KQU( 6343640119895925918), KQU(12878473260854462891), + KQU(10036813920765722626), KQU(14451335468363173812), + KQU( 5476809692401102807), KQU(16442255173514366342), + KQU(13060203194757167104), KQU(14354124071243177715), + KQU(15961249405696125227), KQU(13703893649690872584), + KQU( 363907326340340064), KQU( 6247455540491754842), + KQU(12242249332757832361), KQU( 156065475679796717), + KQU( 9351116235749732355), KQU( 4590350628677701405), + KQU( 1671195940982350389), KQU(13501398458898451905), + KQU( 6526341991225002255), KQU( 1689782913778157592), + KQU( 7439222350869010334), KQU(13975150263226478308), + KQU(11411961169932682710), KQU(17204271834833847277), + KQU( 541534742544435367), KQU( 6591191931218949684), + KQU( 2645454775478232486), KQU( 4322857481256485321), + KQU( 8477416487553065110), KQU(12902505428548435048), + KQU( 971445777981341415), KQU(14995104682744976712), + KQU( 4243341648807158063), KQU( 8695061252721927661), + KQU( 5028202003270177222), KQU( 2289257340915567840), + KQU(13870416345121866007), KQU(13994481698072092233), + KQU( 6912785400753196481), KQU( 2278309315841980139), + KQU( 4329765449648304839), KQU( 5963108095785485298), + KQU( 4880024847478722478), KQU(16015608779890240947), + KQU( 1866679034261393544), KQU( 914821179919731519), + KQU( 9643404035648760131), KQU( 2418114953615593915), + KQU( 944756836073702374), KQU(15186388048737296834), + KQU( 7723355336128442206), KQU( 7500747479679599691), + KQU(18013961306453293634), KQU( 2315274808095756456), + KQU(13655308255424029566), KQU(17203800273561677098), + KQU( 1382158694422087756), KQU( 5090390250309588976), + KQU( 517170818384213989), KQU( 1612709252627729621), + KQU( 1330118955572449606), KQU( 300922478056709885), + KQU(18115693291289091987), KQU(13491407109725238321), + KQU(15293714633593827320), KQU( 5151539373053314504), + KQU( 5951523243743139207), KQU(14459112015249527975), + KQU( 5456113959000700739), KQU( 3877918438464873016), + KQU(12534071654260163555), KQU(15871678376893555041), + KQU(11005484805712025549), KQU(16353066973143374252), + KQU( 4358331472063256685), KQU( 8268349332210859288), + KQU(12485161590939658075), KQU(13955993592854471343), + KQU( 5911446886848367039), KQU(14925834086813706974), + KQU( 6590362597857994805), KQU( 1280544923533661875), + KQU( 1637756018947988164), KQU( 4734090064512686329), + KQU(16693705263131485912), KQU( 6834882340494360958), + KQU( 8120732176159658505), KQU( 2244371958905329346), + KQU(10447499707729734021), KQU( 7318742361446942194), + KQU( 8032857516355555296), KQU(14023605983059313116), + KQU( 1032336061815461376), KQU( 9840995337876562612), + KQU( 9869256223029203587), KQU(12227975697177267636), + KQU(12728115115844186033), KQU( 7752058479783205470), + KQU( 729733219713393087), KQU(12954017801239007622) }; static const uint64_t init_by_array_64_expected[] = { - QU( 2100341266307895239ULL), QU( 8344256300489757943ULL), - QU(15687933285484243894ULL), QU( 8268620370277076319ULL), - QU(12371852309826545459ULL), QU( 8800491541730110238ULL), - QU(18113268950100835773ULL), QU( 2886823658884438119ULL), - QU( 3293667307248180724ULL), QU( 9307928143300172731ULL), - QU( 7688082017574293629ULL), QU( 900986224735166665ULL), - QU( 9977972710722265039ULL), QU( 6008205004994830552ULL), - QU( 546909104521689292ULL), QU( 7428471521869107594ULL), - QU(14777563419314721179ULL), QU(16116143076567350053ULL), - QU( 5322685342003142329ULL), QU( 4200427048445863473ULL), - QU( 4693092150132559146ULL), QU(13671425863759338582ULL), - QU( 6747117460737639916ULL), QU( 4732666080236551150ULL), - QU( 5912839950611941263ULL), QU( 3903717554504704909ULL), - QU( 2615667650256786818ULL), QU(10844129913887006352ULL), - QU(13786467861810997820ULL), QU(14267853002994021570ULL), - QU(13767807302847237439ULL), QU(16407963253707224617ULL), - QU( 4802498363698583497ULL), QU( 2523802839317209764ULL), - QU( 3822579397797475589ULL), QU( 8950320572212130610ULL), - QU( 3745623504978342534ULL), QU(16092609066068482806ULL), - QU( 9817016950274642398ULL), QU(10591660660323829098ULL), - QU(11751606650792815920ULL), QU( 5122873818577122211ULL), - QU(17209553764913936624ULL), QU( 6249057709284380343ULL), - QU(15088791264695071830ULL), QU(15344673071709851930ULL), - QU( 4345751415293646084ULL), QU( 2542865750703067928ULL), - QU(13520525127852368784ULL), QU(18294188662880997241ULL), - QU( 3871781938044881523ULL), QU( 2873487268122812184ULL), - QU(15099676759482679005ULL), QU(15442599127239350490ULL), - QU( 6311893274367710888ULL), QU( 3286118760484672933ULL), - QU( 4146067961333542189ULL), QU(13303942567897208770ULL), - QU( 8196013722255630418ULL), QU( 4437815439340979989ULL), - QU(15433791533450605135ULL), QU( 4254828956815687049ULL), - QU( 1310903207708286015ULL), QU(10529182764462398549ULL), - QU(14900231311660638810ULL), QU( 9727017277104609793ULL), - QU( 1821308310948199033ULL), QU(11628861435066772084ULL), - QU( 9469019138491546924ULL), QU( 3145812670532604988ULL), - QU( 9938468915045491919ULL), QU( 1562447430672662142ULL), - QU(13963995266697989134ULL), QU( 3356884357625028695ULL), - QU( 4499850304584309747ULL), QU( 8456825817023658122ULL), - QU(10859039922814285279ULL), QU( 8099512337972526555ULL), - QU( 348006375109672149ULL), QU(11919893998241688603ULL), - QU( 1104199577402948826ULL), QU(16689191854356060289ULL), - QU(10992552041730168078ULL), QU( 7243733172705465836ULL), - QU( 5668075606180319560ULL), QU(18182847037333286970ULL), - QU( 4290215357664631322ULL), QU( 4061414220791828613ULL), - QU(13006291061652989604ULL), QU( 7140491178917128798ULL), - QU(12703446217663283481ULL), QU( 5500220597564558267ULL), - QU(10330551509971296358ULL), QU(15958554768648714492ULL), - QU( 5174555954515360045ULL), QU( 1731318837687577735ULL), - QU( 3557700801048354857ULL), QU(13764012341928616198ULL), - QU(13115166194379119043ULL), QU( 7989321021560255519ULL), - QU( 2103584280905877040ULL), QU( 9230788662155228488ULL), - QU(16396629323325547654ULL), QU( 657926409811318051ULL), - QU(15046700264391400727ULL), QU( 5120132858771880830ULL), - QU( 7934160097989028561ULL), QU( 6963121488531976245ULL), - QU(17412329602621742089ULL), QU(15144843053931774092ULL), - QU(17204176651763054532ULL), QU(13166595387554065870ULL), - QU( 8590377810513960213ULL), QU( 5834365135373991938ULL), - QU( 7640913007182226243ULL), QU( 3479394703859418425ULL), - QU(16402784452644521040ULL), QU( 4993979809687083980ULL), - QU(13254522168097688865ULL), QU(15643659095244365219ULL), - QU( 5881437660538424982ULL), QU(11174892200618987379ULL), - QU( 254409966159711077ULL), QU(17158413043140549909ULL), - QU( 3638048789290376272ULL), QU( 1376816930299489190ULL), - QU( 4622462095217761923ULL), QU(15086407973010263515ULL), - QU(13253971772784692238ULL), QU( 5270549043541649236ULL), - QU(11182714186805411604ULL), QU(12283846437495577140ULL), - QU( 5297647149908953219ULL), QU(10047451738316836654ULL), - QU( 4938228100367874746ULL), QU(12328523025304077923ULL), - QU( 3601049438595312361ULL), QU( 9313624118352733770ULL), - QU(13322966086117661798ULL), QU(16660005705644029394ULL), - QU(11337677526988872373ULL), QU(13869299102574417795ULL), - QU(15642043183045645437ULL), QU( 3021755569085880019ULL), - QU( 4979741767761188161ULL), QU(13679979092079279587ULL), - QU( 3344685842861071743ULL), QU(13947960059899588104ULL), - QU( 305806934293368007ULL), QU( 5749173929201650029ULL), - QU(11123724852118844098ULL), QU(15128987688788879802ULL), - QU(15251651211024665009ULL), QU( 7689925933816577776ULL), - QU(16732804392695859449ULL), QU(17087345401014078468ULL), - QU(14315108589159048871ULL), QU( 4820700266619778917ULL), - QU(16709637539357958441ULL), QU( 4936227875177351374ULL), - QU( 2137907697912987247ULL), QU(11628565601408395420ULL), - QU( 2333250549241556786ULL), QU( 5711200379577778637ULL), - QU( 5170680131529031729ULL), QU(12620392043061335164ULL), - QU( 95363390101096078ULL), QU( 5487981914081709462ULL), - QU( 1763109823981838620ULL), QU( 3395861271473224396ULL), - QU( 1300496844282213595ULL), QU( 6894316212820232902ULL), - QU(10673859651135576674ULL), QU( 5911839658857903252ULL), - QU(17407110743387299102ULL), QU( 8257427154623140385ULL), - QU(11389003026741800267ULL), QU( 4070043211095013717ULL), - QU(11663806997145259025ULL), QU(15265598950648798210ULL), - QU( 630585789434030934ULL), QU( 3524446529213587334ULL), - QU( 7186424168495184211ULL), QU(10806585451386379021ULL), - QU(11120017753500499273ULL), QU( 1586837651387701301ULL), - QU(17530454400954415544ULL), QU( 9991670045077880430ULL), - QU( 7550997268990730180ULL), QU( 8640249196597379304ULL), - QU( 3522203892786893823ULL), QU(10401116549878854788ULL), - QU(13690285544733124852ULL), QU( 8295785675455774586ULL), - QU(15535716172155117603ULL), QU( 3112108583723722511ULL), - QU(17633179955339271113ULL), QU(18154208056063759375ULL), - QU( 1866409236285815666ULL), QU(13326075895396412882ULL), - QU( 8756261842948020025ULL), QU( 6281852999868439131ULL), - QU(15087653361275292858ULL), QU(10333923911152949397ULL), - QU( 5265567645757408500ULL), QU(12728041843210352184ULL), - QU( 6347959327507828759ULL), QU( 154112802625564758ULL), - QU(18235228308679780218ULL), QU( 3253805274673352418ULL), - QU( 4849171610689031197ULL), QU(17948529398340432518ULL), - QU(13803510475637409167ULL), QU(13506570190409883095ULL), - QU(15870801273282960805ULL), QU( 8451286481299170773ULL), - QU( 9562190620034457541ULL), QU( 8518905387449138364ULL), - QU(12681306401363385655ULL), QU( 3788073690559762558ULL), - QU( 5256820289573487769ULL), QU( 2752021372314875467ULL), - QU( 6354035166862520716ULL), QU( 4328956378309739069ULL), - QU( 449087441228269600ULL), QU( 5533508742653090868ULL), - QU( 1260389420404746988ULL), QU(18175394473289055097ULL), - QU( 1535467109660399420ULL), QU( 8818894282874061442ULL), - QU(12140873243824811213ULL), QU(15031386653823014946ULL), - QU( 1286028221456149232ULL), QU( 6329608889367858784ULL), - QU( 9419654354945132725ULL), QU( 6094576547061672379ULL), - QU(17706217251847450255ULL), QU( 1733495073065878126ULL), - QU(16918923754607552663ULL), QU( 8881949849954945044ULL), - QU(12938977706896313891ULL), QU(14043628638299793407ULL), - QU(18393874581723718233ULL), QU( 6886318534846892044ULL), - QU(14577870878038334081ULL), QU(13541558383439414119ULL), - QU(13570472158807588273ULL), QU(18300760537910283361ULL), - QU( 818368572800609205ULL), QU( 1417000585112573219ULL), - QU(12337533143867683655ULL), QU(12433180994702314480ULL), - QU( 778190005829189083ULL), QU(13667356216206524711ULL), - QU( 9866149895295225230ULL), QU(11043240490417111999ULL), - QU( 1123933826541378598ULL), QU( 6469631933605123610ULL), - QU(14508554074431980040ULL), QU(13918931242962026714ULL), - QU( 2870785929342348285ULL), QU(14786362626740736974ULL), - QU(13176680060902695786ULL), QU( 9591778613541679456ULL), - QU( 9097662885117436706ULL), QU( 749262234240924947ULL), - QU( 1944844067793307093ULL), QU( 4339214904577487742ULL), - QU( 8009584152961946551ULL), QU(16073159501225501777ULL), - QU( 3335870590499306217ULL), QU(17088312653151202847ULL), - QU( 3108893142681931848ULL), QU(16636841767202792021ULL), - QU(10423316431118400637ULL), QU( 8008357368674443506ULL), - QU(11340015231914677875ULL), QU(17687896501594936090ULL), - QU(15173627921763199958ULL), QU( 542569482243721959ULL), - QU(15071714982769812975ULL), QU( 4466624872151386956ULL), - QU( 1901780715602332461ULL), QU( 9822227742154351098ULL), - QU( 1479332892928648780ULL), QU( 6981611948382474400ULL), - QU( 7620824924456077376ULL), QU(14095973329429406782ULL), - QU( 7902744005696185404ULL), QU(15830577219375036920ULL), - QU(10287076667317764416ULL), QU(12334872764071724025ULL), - QU( 4419302088133544331ULL), QU(14455842851266090520ULL), - QU(12488077416504654222ULL), QU( 7953892017701886766ULL), - QU( 6331484925529519007ULL), QU( 4902145853785030022ULL), - QU(17010159216096443073ULL), QU(11945354668653886087ULL), - QU(15112022728645230829ULL), QU(17363484484522986742ULL), - QU( 4423497825896692887ULL), QU( 8155489510809067471ULL), - QU( 258966605622576285ULL), QU( 5462958075742020534ULL), - QU( 6763710214913276228ULL), QU( 2368935183451109054ULL), - QU(14209506165246453811ULL), QU( 2646257040978514881ULL), - QU( 3776001911922207672ULL), QU( 1419304601390147631ULL), - QU(14987366598022458284ULL), QU( 3977770701065815721ULL), - QU( 730820417451838898ULL), QU( 3982991703612885327ULL), - QU( 2803544519671388477ULL), QU(17067667221114424649ULL), - QU( 2922555119737867166ULL), QU( 1989477584121460932ULL), - QU(15020387605892337354ULL), QU( 9293277796427533547ULL), - QU(10722181424063557247ULL), QU(16704542332047511651ULL), - QU( 5008286236142089514ULL), QU(16174732308747382540ULL), - QU(17597019485798338402ULL), QU(13081745199110622093ULL), - QU( 8850305883842258115ULL), QU(12723629125624589005ULL), - QU( 8140566453402805978ULL), QU(15356684607680935061ULL), - QU(14222190387342648650ULL), QU(11134610460665975178ULL), - QU( 1259799058620984266ULL), QU(13281656268025610041ULL), - QU( 298262561068153992ULL), QU(12277871700239212922ULL), - QU(13911297774719779438ULL), QU(16556727962761474934ULL), - QU(17903010316654728010ULL), QU( 9682617699648434744ULL), - QU(14757681836838592850ULL), QU( 1327242446558524473ULL), - QU(11126645098780572792ULL), QU( 1883602329313221774ULL), - QU( 2543897783922776873ULL), QU(15029168513767772842ULL), - QU(12710270651039129878ULL), QU(16118202956069604504ULL), - QU(15010759372168680524ULL), QU( 2296827082251923948ULL), - QU(10793729742623518101ULL), QU(13829764151845413046ULL), - QU(17769301223184451213ULL), QU( 3118268169210783372ULL), - QU(17626204544105123127ULL), QU( 7416718488974352644ULL), - QU(10450751996212925994ULL), QU( 9352529519128770586ULL), - QU( 259347569641110140ULL), QU( 8048588892269692697ULL), - QU( 1774414152306494058ULL), QU(10669548347214355622ULL), - QU(13061992253816795081ULL), QU(18432677803063861659ULL), - QU( 8879191055593984333ULL), QU(12433753195199268041ULL), - QU(14919392415439730602ULL), QU( 6612848378595332963ULL), - QU( 6320986812036143628ULL), QU(10465592420226092859ULL), - QU( 4196009278962570808ULL), QU( 3747816564473572224ULL), - QU(17941203486133732898ULL), QU( 2350310037040505198ULL), - QU( 5811779859134370113ULL), QU(10492109599506195126ULL), - QU( 7699650690179541274ULL), QU( 1954338494306022961ULL), - QU(14095816969027231152ULL), QU( 5841346919964852061ULL), - QU(14945969510148214735ULL), QU( 3680200305887550992ULL), - QU( 6218047466131695792ULL), QU( 8242165745175775096ULL), - QU(11021371934053307357ULL), QU( 1265099502753169797ULL), - QU( 4644347436111321718ULL), QU( 3609296916782832859ULL), - QU( 8109807992218521571ULL), QU(18387884215648662020ULL), - QU(14656324896296392902ULL), QU(17386819091238216751ULL), - QU(17788300878582317152ULL), QU( 7919446259742399591ULL), - QU( 4466613134576358004ULL), QU(12928181023667938509ULL), - QU(13147446154454932030ULL), QU(16552129038252734620ULL), - QU( 8395299403738822450ULL), QU(11313817655275361164ULL), - QU( 434258809499511718ULL), QU( 2074882104954788676ULL), - QU( 7929892178759395518ULL), QU( 9006461629105745388ULL), - QU( 5176475650000323086ULL), QU(11128357033468341069ULL), - QU(12026158851559118955ULL), QU(14699716249471156500ULL), - QU( 448982497120206757ULL), QU( 4156475356685519900ULL), - QU( 6063816103417215727ULL), QU(10073289387954971479ULL), - QU( 8174466846138590962ULL), QU( 2675777452363449006ULL), - QU( 9090685420572474281ULL), QU( 6659652652765562060ULL), - QU(12923120304018106621ULL), QU(11117480560334526775ULL), - QU( 937910473424587511ULL), QU( 1838692113502346645ULL), - QU(11133914074648726180ULL), QU( 7922600945143884053ULL), - QU(13435287702700959550ULL), QU( 5287964921251123332ULL), - QU(11354875374575318947ULL), QU(17955724760748238133ULL), - QU(13728617396297106512ULL), QU( 4107449660118101255ULL), - QU( 1210269794886589623ULL), QU(11408687205733456282ULL), - QU( 4538354710392677887ULL), QU(13566803319341319267ULL), - QU(17870798107734050771ULL), QU( 3354318982568089135ULL), - QU( 9034450839405133651ULL), QU(13087431795753424314ULL), - QU( 950333102820688239ULL), QU( 1968360654535604116ULL), - QU(16840551645563314995ULL), QU( 8867501803892924995ULL), - QU(11395388644490626845ULL), QU( 1529815836300732204ULL), - QU(13330848522996608842ULL), QU( 1813432878817504265ULL), - QU( 2336867432693429560ULL), QU(15192805445973385902ULL), - QU( 2528593071076407877ULL), QU( 128459777936689248ULL), - QU( 9976345382867214866ULL), QU( 6208885766767996043ULL), - QU(14982349522273141706ULL), QU( 3099654362410737822ULL), - QU(13776700761947297661ULL), QU( 8806185470684925550ULL), - QU( 8151717890410585321ULL), QU( 640860591588072925ULL), - QU(14592096303937307465ULL), QU( 9056472419613564846ULL), - QU(14861544647742266352ULL), QU(12703771500398470216ULL), - QU( 3142372800384138465ULL), QU( 6201105606917248196ULL), - QU(18337516409359270184ULL), QU(15042268695665115339ULL), - QU(15188246541383283846ULL), QU(12800028693090114519ULL), - QU( 5992859621101493472ULL), QU(18278043971816803521ULL), - QU( 9002773075219424560ULL), QU( 7325707116943598353ULL), - QU( 7930571931248040822ULL), QU( 5645275869617023448ULL), - QU( 7266107455295958487ULL), QU( 4363664528273524411ULL), - QU(14313875763787479809ULL), QU(17059695613553486802ULL), - QU( 9247761425889940932ULL), QU(13704726459237593128ULL), - QU( 2701312427328909832ULL), QU(17235532008287243115ULL), - QU(14093147761491729538ULL), QU( 6247352273768386516ULL), - QU( 8268710048153268415ULL), QU( 7985295214477182083ULL), - QU(15624495190888896807ULL), QU( 3772753430045262788ULL), - QU( 9133991620474991698ULL), QU( 5665791943316256028ULL), - QU( 7551996832462193473ULL), QU(13163729206798953877ULL), - QU( 9263532074153846374ULL), QU( 1015460703698618353ULL), - QU(17929874696989519390ULL), QU(18257884721466153847ULL), - QU(16271867543011222991ULL), QU( 3905971519021791941ULL), - QU(16814488397137052085ULL), QU( 1321197685504621613ULL), - QU( 2870359191894002181ULL), QU(14317282970323395450ULL), - QU(13663920845511074366ULL), QU( 2052463995796539594ULL), - QU(14126345686431444337ULL), QU( 1727572121947022534ULL), - QU(17793552254485594241ULL), QU( 6738857418849205750ULL), - QU( 1282987123157442952ULL), QU(16655480021581159251ULL), - QU( 6784587032080183866ULL), QU(14726758805359965162ULL), - QU( 7577995933961987349ULL), QU(12539609320311114036ULL), - QU(10789773033385439494ULL), QU( 8517001497411158227ULL), - QU(10075543932136339710ULL), QU(14838152340938811081ULL), - QU( 9560840631794044194ULL), QU(17445736541454117475ULL), - QU(10633026464336393186ULL), QU(15705729708242246293ULL), - QU( 1117517596891411098ULL), QU( 4305657943415886942ULL), - QU( 4948856840533979263ULL), QU(16071681989041789593ULL), - QU(13723031429272486527ULL), QU( 7639567622306509462ULL), - QU(12670424537483090390ULL), QU( 9715223453097197134ULL), - QU( 5457173389992686394ULL), QU( 289857129276135145ULL), - QU(17048610270521972512ULL), QU( 692768013309835485ULL), - QU(14823232360546632057ULL), QU(18218002361317895936ULL), - QU( 3281724260212650204ULL), QU(16453957266549513795ULL), - QU( 8592711109774511881ULL), QU( 929825123473369579ULL), - QU(15966784769764367791ULL), QU( 9627344291450607588ULL), - QU(10849555504977813287ULL), QU( 9234566913936339275ULL), - QU( 6413807690366911210ULL), QU(10862389016184219267ULL), - QU(13842504799335374048ULL), QU( 1531994113376881174ULL), - QU( 2081314867544364459ULL), QU(16430628791616959932ULL), - QU( 8314714038654394368ULL), QU( 9155473892098431813ULL), - QU(12577843786670475704ULL), QU( 4399161106452401017ULL), - QU( 1668083091682623186ULL), QU( 1741383777203714216ULL), - QU( 2162597285417794374ULL), QU(15841980159165218736ULL), - QU( 1971354603551467079ULL), QU( 1206714764913205968ULL), - QU( 4790860439591272330ULL), QU(14699375615594055799ULL), - QU( 8374423871657449988ULL), QU(10950685736472937738ULL), - QU( 697344331343267176ULL), QU(10084998763118059810ULL), - QU(12897369539795983124ULL), QU(12351260292144383605ULL), - QU( 1268810970176811234ULL), QU( 7406287800414582768ULL), - QU( 516169557043807831ULL), QU( 5077568278710520380ULL), - QU( 3828791738309039304ULL), QU( 7721974069946943610ULL), - QU( 3534670260981096460ULL), QU( 4865792189600584891ULL), - QU(16892578493734337298ULL), QU( 9161499464278042590ULL), - QU(11976149624067055931ULL), QU(13219479887277343990ULL), - QU(14161556738111500680ULL), QU(14670715255011223056ULL), - QU( 4671205678403576558ULL), QU(12633022931454259781ULL), - QU(14821376219869187646ULL), QU( 751181776484317028ULL), - QU( 2192211308839047070ULL), QU(11787306362361245189ULL), - QU(10672375120744095707ULL), QU( 4601972328345244467ULL), - QU(15457217788831125879ULL), QU( 8464345256775460809ULL), - QU(10191938789487159478ULL), QU( 6184348739615197613ULL), - QU(11425436778806882100ULL), QU( 2739227089124319793ULL), - QU( 461464518456000551ULL), QU( 4689850170029177442ULL), - QU( 6120307814374078625ULL), QU(11153579230681708671ULL), - QU( 7891721473905347926ULL), QU(10281646937824872400ULL), - QU( 3026099648191332248ULL), QU( 8666750296953273818ULL), - QU(14978499698844363232ULL), QU(13303395102890132065ULL), - QU( 8182358205292864080ULL), QU(10560547713972971291ULL), - QU(11981635489418959093ULL), QU( 3134621354935288409ULL), - QU(11580681977404383968ULL), QU(14205530317404088650ULL), - QU( 5997789011854923157ULL), QU(13659151593432238041ULL), - QU(11664332114338865086ULL), QU( 7490351383220929386ULL), - QU( 7189290499881530378ULL), QU(15039262734271020220ULL), - QU( 2057217285976980055ULL), QU( 555570804905355739ULL), - QU(11235311968348555110ULL), QU(13824557146269603217ULL), - QU(16906788840653099693ULL), QU( 7222878245455661677ULL), - QU( 5245139444332423756ULL), QU( 4723748462805674292ULL), - QU(12216509815698568612ULL), QU(17402362976648951187ULL), - QU(17389614836810366768ULL), QU( 4880936484146667711ULL), - QU( 9085007839292639880ULL), QU(13837353458498535449ULL), - QU(11914419854360366677ULL), QU(16595890135313864103ULL), - QU( 6313969847197627222ULL), QU(18296909792163910431ULL), - QU(10041780113382084042ULL), QU( 2499478551172884794ULL), - QU(11057894246241189489ULL), QU( 9742243032389068555ULL), - QU(12838934582673196228ULL), QU(13437023235248490367ULL), - QU(13372420669446163240ULL), QU( 6752564244716909224ULL), - QU( 7157333073400313737ULL), QU(12230281516370654308ULL), - QU( 1182884552219419117ULL), QU( 2955125381312499218ULL), - QU(10308827097079443249ULL), QU( 1337648572986534958ULL), - QU(16378788590020343939ULL), QU( 108619126514420935ULL), - QU( 3990981009621629188ULL), QU( 5460953070230946410ULL), - QU( 9703328329366531883ULL), QU(13166631489188077236ULL), - QU( 1104768831213675170ULL), QU( 3447930458553877908ULL), - QU( 8067172487769945676ULL), QU( 5445802098190775347ULL), - QU( 3244840981648973873ULL), QU(17314668322981950060ULL), - QU( 5006812527827763807ULL), QU(18158695070225526260ULL), - QU( 2824536478852417853ULL), QU(13974775809127519886ULL), - QU( 9814362769074067392ULL), QU(17276205156374862128ULL), - QU(11361680725379306967ULL), QU( 3422581970382012542ULL), - QU(11003189603753241266ULL), QU(11194292945277862261ULL), - QU( 6839623313908521348ULL), QU(11935326462707324634ULL), - QU( 1611456788685878444ULL), QU(13112620989475558907ULL), - QU( 517659108904450427ULL), QU(13558114318574407624ULL), - QU(15699089742731633077ULL), QU( 4988979278862685458ULL), - QU( 8111373583056521297ULL), QU( 3891258746615399627ULL), - QU( 8137298251469718086ULL), QU(12748663295624701649ULL), - QU( 4389835683495292062ULL), QU( 5775217872128831729ULL), - QU( 9462091896405534927ULL), QU( 8498124108820263989ULL), - QU( 8059131278842839525ULL), QU(10503167994254090892ULL), - QU(11613153541070396656ULL), QU(18069248738504647790ULL), - QU( 570657419109768508ULL), QU( 3950574167771159665ULL), - QU( 5514655599604313077ULL), QU( 2908460854428484165ULL), - QU(10777722615935663114ULL), QU(12007363304839279486ULL), - QU( 9800646187569484767ULL), QU( 8795423564889864287ULL), - QU(14257396680131028419ULL), QU( 6405465117315096498ULL), - QU( 7939411072208774878ULL), QU(17577572378528990006ULL), - QU(14785873806715994850ULL), QU(16770572680854747390ULL), - QU(18127549474419396481ULL), QU(11637013449455757750ULL), - QU(14371851933996761086ULL), QU( 3601181063650110280ULL), - QU( 4126442845019316144ULL), QU(10198287239244320669ULL), - QU(18000169628555379659ULL), QU(18392482400739978269ULL), - QU( 6219919037686919957ULL), QU( 3610085377719446052ULL), - QU( 2513925039981776336ULL), QU(16679413537926716955ULL), - QU(12903302131714909434ULL), QU( 5581145789762985009ULL), - QU(12325955044293303233ULL), QU(17216111180742141204ULL), - QU( 6321919595276545740ULL), QU( 3507521147216174501ULL), - QU( 9659194593319481840ULL), QU(11473976005975358326ULL), - QU(14742730101435987026ULL), QU( 492845897709954780ULL), - QU(16976371186162599676ULL), QU(17712703422837648655ULL), - QU( 9881254778587061697ULL), QU( 8413223156302299551ULL), - QU( 1563841828254089168ULL), QU( 9996032758786671975ULL), - QU( 138877700583772667ULL), QU(13003043368574995989ULL), - QU( 4390573668650456587ULL), QU( 8610287390568126755ULL), - QU(15126904974266642199ULL), QU( 6703637238986057662ULL), - QU( 2873075592956810157ULL), QU( 6035080933946049418ULL), - QU(13382846581202353014ULL), QU( 7303971031814642463ULL), - QU(18418024405307444267ULL), QU( 5847096731675404647ULL), - QU( 4035880699639842500ULL), QU(11525348625112218478ULL), - QU( 3041162365459574102ULL), QU( 2604734487727986558ULL), - QU(15526341771636983145ULL), QU(14556052310697370254ULL), - QU(12997787077930808155ULL), QU( 9601806501755554499ULL), - QU(11349677952521423389ULL), QU(14956777807644899350ULL), - QU(16559736957742852721ULL), QU(12360828274778140726ULL), - QU( 6685373272009662513ULL), QU(16932258748055324130ULL), - QU(15918051131954158508ULL), QU( 1692312913140790144ULL), - QU( 546653826801637367ULL), QU( 5341587076045986652ULL), - QU(14975057236342585662ULL), QU(12374976357340622412ULL), - QU(10328833995181940552ULL), QU(12831807101710443149ULL), - QU(10548514914382545716ULL), QU( 2217806727199715993ULL), - QU(12627067369242845138ULL), QU( 4598965364035438158ULL), - QU( 150923352751318171ULL), QU(14274109544442257283ULL), - QU( 4696661475093863031ULL), QU( 1505764114384654516ULL), - QU(10699185831891495147ULL), QU( 2392353847713620519ULL), - QU( 3652870166711788383ULL), QU( 8640653276221911108ULL), - QU( 3894077592275889704ULL), QU( 4918592872135964845ULL), - QU(16379121273281400789ULL), QU(12058465483591683656ULL), - QU(11250106829302924945ULL), QU( 1147537556296983005ULL), - QU( 6376342756004613268ULL), QU(14967128191709280506ULL), - QU(18007449949790627628ULL), QU( 9497178279316537841ULL), - QU( 7920174844809394893ULL), QU(10037752595255719907ULL), - QU(15875342784985217697ULL), QU(15311615921712850696ULL), - QU( 9552902652110992950ULL), QU(14054979450099721140ULL), - QU( 5998709773566417349ULL), QU(18027910339276320187ULL), - QU( 8223099053868585554ULL), QU( 7842270354824999767ULL), - QU( 4896315688770080292ULL), QU(12969320296569787895ULL), - QU( 2674321489185759961ULL), QU( 4053615936864718439ULL), - QU(11349775270588617578ULL), QU( 4743019256284553975ULL), - QU( 5602100217469723769ULL), QU(14398995691411527813ULL), - QU( 7412170493796825470ULL), QU( 836262406131744846ULL), - QU( 8231086633845153022ULL), QU( 5161377920438552287ULL), - QU( 8828731196169924949ULL), QU(16211142246465502680ULL), - QU( 3307990879253687818ULL), QU( 5193405406899782022ULL), - QU( 8510842117467566693ULL), QU( 6070955181022405365ULL), - QU(14482950231361409799ULL), QU(12585159371331138077ULL), - QU( 3511537678933588148ULL), QU( 2041849474531116417ULL), - QU(10944936685095345792ULL), QU(18303116923079107729ULL), - QU( 2720566371239725320ULL), QU( 4958672473562397622ULL), - QU( 3032326668253243412ULL), QU(13689418691726908338ULL), - QU( 1895205511728843996ULL), QU( 8146303515271990527ULL), - QU(16507343500056113480ULL), QU( 473996939105902919ULL), - QU( 9897686885246881481ULL), QU(14606433762712790575ULL), - QU( 6732796251605566368ULL), QU( 1399778120855368916ULL), - QU( 935023885182833777ULL), QU(16066282816186753477ULL), - QU( 7291270991820612055ULL), QU(17530230393129853844ULL), - QU(10223493623477451366ULL), QU(15841725630495676683ULL), - QU(17379567246435515824ULL), QU( 8588251429375561971ULL), - QU(18339511210887206423ULL), QU(17349587430725976100ULL), - QU(12244876521394838088ULL), QU( 6382187714147161259ULL), - QU(12335807181848950831ULL), QU(16948885622305460665ULL), - QU(13755097796371520506ULL), QU(14806740373324947801ULL), - QU( 4828699633859287703ULL), QU( 8209879281452301604ULL), - QU(12435716669553736437ULL), QU(13970976859588452131ULL), - QU( 6233960842566773148ULL), QU(12507096267900505759ULL), - QU( 1198713114381279421ULL), QU(14989862731124149015ULL), - QU(15932189508707978949ULL), QU( 2526406641432708722ULL), - QU( 29187427817271982ULL), QU( 1499802773054556353ULL), - QU(10816638187021897173ULL), QU( 5436139270839738132ULL), - QU( 6659882287036010082ULL), QU( 2154048955317173697ULL), - QU(10887317019333757642ULL), QU(16281091802634424955ULL), - QU(10754549879915384901ULL), QU(10760611745769249815ULL), - QU( 2161505946972504002ULL), QU( 5243132808986265107ULL), - QU(10129852179873415416ULL), QU( 710339480008649081ULL), - QU( 7802129453068808528ULL), QU(17967213567178907213ULL), - QU(15730859124668605599ULL), QU(13058356168962376502ULL), - QU( 3701224985413645909ULL), QU(14464065869149109264ULL), - QU( 9959272418844311646ULL), QU(10157426099515958752ULL), - QU(14013736814538268528ULL), QU(17797456992065653951ULL), - QU(17418878140257344806ULL), QU(15457429073540561521ULL), - QU( 2184426881360949378ULL), QU( 2062193041154712416ULL), - QU( 8553463347406931661ULL), QU( 4913057625202871854ULL), - QU( 2668943682126618425ULL), QU(17064444737891172288ULL), - QU( 4997115903913298637ULL), QU(12019402608892327416ULL), - QU(17603584559765897352ULL), QU(11367529582073647975ULL), - QU( 8211476043518436050ULL), QU( 8676849804070323674ULL), - QU(18431829230394475730ULL), QU(10490177861361247904ULL), - QU( 9508720602025651349ULL), QU( 7409627448555722700ULL), - QU( 5804047018862729008ULL), QU(11943858176893142594ULL), - QU(11908095418933847092ULL), QU( 5415449345715887652ULL), - QU( 1554022699166156407ULL), QU( 9073322106406017161ULL), - QU( 7080630967969047082ULL), QU(18049736940860732943ULL), - QU(12748714242594196794ULL), QU( 1226992415735156741ULL), - QU(17900981019609531193ULL), QU(11720739744008710999ULL), - QU( 3006400683394775434ULL), QU(11347974011751996028ULL), - QU( 3316999628257954608ULL), QU( 8384484563557639101ULL), - QU(18117794685961729767ULL), QU( 1900145025596618194ULL), - QU(17459527840632892676ULL), QU( 5634784101865710994ULL), - QU( 7918619300292897158ULL), QU( 3146577625026301350ULL), - QU( 9955212856499068767ULL), QU( 1873995843681746975ULL), - QU( 1561487759967972194ULL), QU( 8322718804375878474ULL), - QU(11300284215327028366ULL), QU( 4667391032508998982ULL), - QU( 9820104494306625580ULL), QU(17922397968599970610ULL), - QU( 1784690461886786712ULL), QU(14940365084341346821ULL), - QU( 5348719575594186181ULL), QU(10720419084507855261ULL), - QU(14210394354145143274ULL), QU( 2426468692164000131ULL), - QU(16271062114607059202ULL), QU(14851904092357070247ULL), - QU( 6524493015693121897ULL), QU( 9825473835127138531ULL), - QU(14222500616268569578ULL), QU(15521484052007487468ULL), - QU(14462579404124614699ULL), QU(11012375590820665520ULL), - QU(11625327350536084927ULL), QU(14452017765243785417ULL), - QU( 9989342263518766305ULL), QU( 3640105471101803790ULL), - QU( 4749866455897513242ULL), QU(13963064946736312044ULL), - QU(10007416591973223791ULL), QU(18314132234717431115ULL), - QU( 3286596588617483450ULL), QU( 7726163455370818765ULL), - QU( 7575454721115379328ULL), QU( 5308331576437663422ULL), - QU(18288821894903530934ULL), QU( 8028405805410554106ULL), - QU(15744019832103296628ULL), QU( 149765559630932100ULL), - QU( 6137705557200071977ULL), QU(14513416315434803615ULL), - QU(11665702820128984473ULL), QU( 218926670505601386ULL), - QU( 6868675028717769519ULL), QU(15282016569441512302ULL), - QU( 5707000497782960236ULL), QU( 6671120586555079567ULL), - QU( 2194098052618985448ULL), QU(16849577895477330978ULL), - QU(12957148471017466283ULL), QU( 1997805535404859393ULL), - QU( 1180721060263860490ULL), QU(13206391310193756958ULL), - QU(12980208674461861797ULL), QU( 3825967775058875366ULL), - QU(17543433670782042631ULL), QU( 1518339070120322730ULL), - QU(16344584340890991669ULL), QU( 2611327165318529819ULL), - QU(11265022723283422529ULL), QU( 4001552800373196817ULL), - QU(14509595890079346161ULL), QU( 3528717165416234562ULL), - QU(18153222571501914072ULL), QU( 9387182977209744425ULL), - QU(10064342315985580021ULL), QU(11373678413215253977ULL), - QU( 2308457853228798099ULL), QU( 9729042942839545302ULL), - QU( 7833785471140127746ULL), QU( 6351049900319844436ULL), - QU(14454610627133496067ULL), QU(12533175683634819111ULL), - QU(15570163926716513029ULL), QU(13356980519185762498ULL) + KQU( 2100341266307895239), KQU( 8344256300489757943), + KQU(15687933285484243894), KQU( 8268620370277076319), + KQU(12371852309826545459), KQU( 8800491541730110238), + KQU(18113268950100835773), KQU( 2886823658884438119), + KQU( 3293667307248180724), KQU( 9307928143300172731), + KQU( 7688082017574293629), KQU( 900986224735166665), + KQU( 9977972710722265039), KQU( 6008205004994830552), + KQU( 546909104521689292), KQU( 7428471521869107594), + KQU(14777563419314721179), KQU(16116143076567350053), + KQU( 5322685342003142329), KQU( 4200427048445863473), + KQU( 4693092150132559146), KQU(13671425863759338582), + KQU( 6747117460737639916), KQU( 4732666080236551150), + KQU( 5912839950611941263), KQU( 3903717554504704909), + KQU( 2615667650256786818), KQU(10844129913887006352), + KQU(13786467861810997820), KQU(14267853002994021570), + KQU(13767807302847237439), KQU(16407963253707224617), + KQU( 4802498363698583497), KQU( 2523802839317209764), + KQU( 3822579397797475589), KQU( 8950320572212130610), + KQU( 3745623504978342534), KQU(16092609066068482806), + KQU( 9817016950274642398), KQU(10591660660323829098), + KQU(11751606650792815920), KQU( 5122873818577122211), + KQU(17209553764913936624), KQU( 6249057709284380343), + KQU(15088791264695071830), KQU(15344673071709851930), + KQU( 4345751415293646084), KQU( 2542865750703067928), + KQU(13520525127852368784), KQU(18294188662880997241), + KQU( 3871781938044881523), KQU( 2873487268122812184), + KQU(15099676759482679005), KQU(15442599127239350490), + KQU( 6311893274367710888), KQU( 3286118760484672933), + KQU( 4146067961333542189), KQU(13303942567897208770), + KQU( 8196013722255630418), KQU( 4437815439340979989), + KQU(15433791533450605135), KQU( 4254828956815687049), + KQU( 1310903207708286015), KQU(10529182764462398549), + KQU(14900231311660638810), KQU( 9727017277104609793), + KQU( 1821308310948199033), KQU(11628861435066772084), + KQU( 9469019138491546924), KQU( 3145812670532604988), + KQU( 9938468915045491919), KQU( 1562447430672662142), + KQU(13963995266697989134), KQU( 3356884357625028695), + KQU( 4499850304584309747), KQU( 8456825817023658122), + KQU(10859039922814285279), KQU( 8099512337972526555), + KQU( 348006375109672149), KQU(11919893998241688603), + KQU( 1104199577402948826), KQU(16689191854356060289), + KQU(10992552041730168078), KQU( 7243733172705465836), + KQU( 5668075606180319560), KQU(18182847037333286970), + KQU( 4290215357664631322), KQU( 4061414220791828613), + KQU(13006291061652989604), KQU( 7140491178917128798), + KQU(12703446217663283481), KQU( 5500220597564558267), + KQU(10330551509971296358), KQU(15958554768648714492), + KQU( 5174555954515360045), KQU( 1731318837687577735), + KQU( 3557700801048354857), KQU(13764012341928616198), + KQU(13115166194379119043), KQU( 7989321021560255519), + KQU( 2103584280905877040), KQU( 9230788662155228488), + KQU(16396629323325547654), KQU( 657926409811318051), + KQU(15046700264391400727), KQU( 5120132858771880830), + KQU( 7934160097989028561), KQU( 6963121488531976245), + KQU(17412329602621742089), KQU(15144843053931774092), + KQU(17204176651763054532), KQU(13166595387554065870), + KQU( 8590377810513960213), KQU( 5834365135373991938), + KQU( 7640913007182226243), KQU( 3479394703859418425), + KQU(16402784452644521040), KQU( 4993979809687083980), + KQU(13254522168097688865), KQU(15643659095244365219), + KQU( 5881437660538424982), KQU(11174892200618987379), + KQU( 254409966159711077), KQU(17158413043140549909), + KQU( 3638048789290376272), KQU( 1376816930299489190), + KQU( 4622462095217761923), KQU(15086407973010263515), + KQU(13253971772784692238), KQU( 5270549043541649236), + KQU(11182714186805411604), KQU(12283846437495577140), + KQU( 5297647149908953219), KQU(10047451738316836654), + KQU( 4938228100367874746), KQU(12328523025304077923), + KQU( 3601049438595312361), KQU( 9313624118352733770), + KQU(13322966086117661798), KQU(16660005705644029394), + KQU(11337677526988872373), KQU(13869299102574417795), + KQU(15642043183045645437), KQU( 3021755569085880019), + KQU( 4979741767761188161), KQU(13679979092079279587), + KQU( 3344685842861071743), KQU(13947960059899588104), + KQU( 305806934293368007), KQU( 5749173929201650029), + KQU(11123724852118844098), KQU(15128987688788879802), + KQU(15251651211024665009), KQU( 7689925933816577776), + KQU(16732804392695859449), KQU(17087345401014078468), + KQU(14315108589159048871), KQU( 4820700266619778917), + KQU(16709637539357958441), KQU( 4936227875177351374), + KQU( 2137907697912987247), KQU(11628565601408395420), + KQU( 2333250549241556786), KQU( 5711200379577778637), + KQU( 5170680131529031729), KQU(12620392043061335164), + KQU( 95363390101096078), KQU( 5487981914081709462), + KQU( 1763109823981838620), KQU( 3395861271473224396), + KQU( 1300496844282213595), KQU( 6894316212820232902), + KQU(10673859651135576674), KQU( 5911839658857903252), + KQU(17407110743387299102), KQU( 8257427154623140385), + KQU(11389003026741800267), KQU( 4070043211095013717), + KQU(11663806997145259025), KQU(15265598950648798210), + KQU( 630585789434030934), KQU( 3524446529213587334), + KQU( 7186424168495184211), KQU(10806585451386379021), + KQU(11120017753500499273), KQU( 1586837651387701301), + KQU(17530454400954415544), KQU( 9991670045077880430), + KQU( 7550997268990730180), KQU( 8640249196597379304), + KQU( 3522203892786893823), KQU(10401116549878854788), + KQU(13690285544733124852), KQU( 8295785675455774586), + KQU(15535716172155117603), KQU( 3112108583723722511), + KQU(17633179955339271113), KQU(18154208056063759375), + KQU( 1866409236285815666), KQU(13326075895396412882), + KQU( 8756261842948020025), KQU( 6281852999868439131), + KQU(15087653361275292858), KQU(10333923911152949397), + KQU( 5265567645757408500), KQU(12728041843210352184), + KQU( 6347959327507828759), KQU( 154112802625564758), + KQU(18235228308679780218), KQU( 3253805274673352418), + KQU( 4849171610689031197), KQU(17948529398340432518), + KQU(13803510475637409167), KQU(13506570190409883095), + KQU(15870801273282960805), KQU( 8451286481299170773), + KQU( 9562190620034457541), KQU( 8518905387449138364), + KQU(12681306401363385655), KQU( 3788073690559762558), + KQU( 5256820289573487769), KQU( 2752021372314875467), + KQU( 6354035166862520716), KQU( 4328956378309739069), + KQU( 449087441228269600), KQU( 5533508742653090868), + KQU( 1260389420404746988), KQU(18175394473289055097), + KQU( 1535467109660399420), KQU( 8818894282874061442), + KQU(12140873243824811213), KQU(15031386653823014946), + KQU( 1286028221456149232), KQU( 6329608889367858784), + KQU( 9419654354945132725), KQU( 6094576547061672379), + KQU(17706217251847450255), KQU( 1733495073065878126), + KQU(16918923754607552663), KQU( 8881949849954945044), + KQU(12938977706896313891), KQU(14043628638299793407), + KQU(18393874581723718233), KQU( 6886318534846892044), + KQU(14577870878038334081), KQU(13541558383439414119), + KQU(13570472158807588273), KQU(18300760537910283361), + KQU( 818368572800609205), KQU( 1417000585112573219), + KQU(12337533143867683655), KQU(12433180994702314480), + KQU( 778190005829189083), KQU(13667356216206524711), + KQU( 9866149895295225230), KQU(11043240490417111999), + KQU( 1123933826541378598), KQU( 6469631933605123610), + KQU(14508554074431980040), KQU(13918931242962026714), + KQU( 2870785929342348285), KQU(14786362626740736974), + KQU(13176680060902695786), KQU( 9591778613541679456), + KQU( 9097662885117436706), KQU( 749262234240924947), + KQU( 1944844067793307093), KQU( 4339214904577487742), + KQU( 8009584152961946551), KQU(16073159501225501777), + KQU( 3335870590499306217), KQU(17088312653151202847), + KQU( 3108893142681931848), KQU(16636841767202792021), + KQU(10423316431118400637), KQU( 8008357368674443506), + KQU(11340015231914677875), KQU(17687896501594936090), + KQU(15173627921763199958), KQU( 542569482243721959), + KQU(15071714982769812975), KQU( 4466624872151386956), + KQU( 1901780715602332461), KQU( 9822227742154351098), + KQU( 1479332892928648780), KQU( 6981611948382474400), + KQU( 7620824924456077376), KQU(14095973329429406782), + KQU( 7902744005696185404), KQU(15830577219375036920), + KQU(10287076667317764416), KQU(12334872764071724025), + KQU( 4419302088133544331), KQU(14455842851266090520), + KQU(12488077416504654222), KQU( 7953892017701886766), + KQU( 6331484925529519007), KQU( 4902145853785030022), + KQU(17010159216096443073), KQU(11945354668653886087), + KQU(15112022728645230829), KQU(17363484484522986742), + KQU( 4423497825896692887), KQU( 8155489510809067471), + KQU( 258966605622576285), KQU( 5462958075742020534), + KQU( 6763710214913276228), KQU( 2368935183451109054), + KQU(14209506165246453811), KQU( 2646257040978514881), + KQU( 3776001911922207672), KQU( 1419304601390147631), + KQU(14987366598022458284), KQU( 3977770701065815721), + KQU( 730820417451838898), KQU( 3982991703612885327), + KQU( 2803544519671388477), KQU(17067667221114424649), + KQU( 2922555119737867166), KQU( 1989477584121460932), + KQU(15020387605892337354), KQU( 9293277796427533547), + KQU(10722181424063557247), KQU(16704542332047511651), + KQU( 5008286236142089514), KQU(16174732308747382540), + KQU(17597019485798338402), KQU(13081745199110622093), + KQU( 8850305883842258115), KQU(12723629125624589005), + KQU( 8140566453402805978), KQU(15356684607680935061), + KQU(14222190387342648650), KQU(11134610460665975178), + KQU( 1259799058620984266), KQU(13281656268025610041), + KQU( 298262561068153992), KQU(12277871700239212922), + KQU(13911297774719779438), KQU(16556727962761474934), + KQU(17903010316654728010), KQU( 9682617699648434744), + KQU(14757681836838592850), KQU( 1327242446558524473), + KQU(11126645098780572792), KQU( 1883602329313221774), + KQU( 2543897783922776873), KQU(15029168513767772842), + KQU(12710270651039129878), KQU(16118202956069604504), + KQU(15010759372168680524), KQU( 2296827082251923948), + KQU(10793729742623518101), KQU(13829764151845413046), + KQU(17769301223184451213), KQU( 3118268169210783372), + KQU(17626204544105123127), KQU( 7416718488974352644), + KQU(10450751996212925994), KQU( 9352529519128770586), + KQU( 259347569641110140), KQU( 8048588892269692697), + KQU( 1774414152306494058), KQU(10669548347214355622), + KQU(13061992253816795081), KQU(18432677803063861659), + KQU( 8879191055593984333), KQU(12433753195199268041), + KQU(14919392415439730602), KQU( 6612848378595332963), + KQU( 6320986812036143628), KQU(10465592420226092859), + KQU( 4196009278962570808), KQU( 3747816564473572224), + KQU(17941203486133732898), KQU( 2350310037040505198), + KQU( 5811779859134370113), KQU(10492109599506195126), + KQU( 7699650690179541274), KQU( 1954338494306022961), + KQU(14095816969027231152), KQU( 5841346919964852061), + KQU(14945969510148214735), KQU( 3680200305887550992), + KQU( 6218047466131695792), KQU( 8242165745175775096), + KQU(11021371934053307357), KQU( 1265099502753169797), + KQU( 4644347436111321718), KQU( 3609296916782832859), + KQU( 8109807992218521571), KQU(18387884215648662020), + KQU(14656324896296392902), KQU(17386819091238216751), + KQU(17788300878582317152), KQU( 7919446259742399591), + KQU( 4466613134576358004), KQU(12928181023667938509), + KQU(13147446154454932030), KQU(16552129038252734620), + KQU( 8395299403738822450), KQU(11313817655275361164), + KQU( 434258809499511718), KQU( 2074882104954788676), + KQU( 7929892178759395518), KQU( 9006461629105745388), + KQU( 5176475650000323086), KQU(11128357033468341069), + KQU(12026158851559118955), KQU(14699716249471156500), + KQU( 448982497120206757), KQU( 4156475356685519900), + KQU( 6063816103417215727), KQU(10073289387954971479), + KQU( 8174466846138590962), KQU( 2675777452363449006), + KQU( 9090685420572474281), KQU( 6659652652765562060), + KQU(12923120304018106621), KQU(11117480560334526775), + KQU( 937910473424587511), KQU( 1838692113502346645), + KQU(11133914074648726180), KQU( 7922600945143884053), + KQU(13435287702700959550), KQU( 5287964921251123332), + KQU(11354875374575318947), KQU(17955724760748238133), + KQU(13728617396297106512), KQU( 4107449660118101255), + KQU( 1210269794886589623), KQU(11408687205733456282), + KQU( 4538354710392677887), KQU(13566803319341319267), + KQU(17870798107734050771), KQU( 3354318982568089135), + KQU( 9034450839405133651), KQU(13087431795753424314), + KQU( 950333102820688239), KQU( 1968360654535604116), + KQU(16840551645563314995), KQU( 8867501803892924995), + KQU(11395388644490626845), KQU( 1529815836300732204), + KQU(13330848522996608842), KQU( 1813432878817504265), + KQU( 2336867432693429560), KQU(15192805445973385902), + KQU( 2528593071076407877), KQU( 128459777936689248), + KQU( 9976345382867214866), KQU( 6208885766767996043), + KQU(14982349522273141706), KQU( 3099654362410737822), + KQU(13776700761947297661), KQU( 8806185470684925550), + KQU( 8151717890410585321), KQU( 640860591588072925), + KQU(14592096303937307465), KQU( 9056472419613564846), + KQU(14861544647742266352), KQU(12703771500398470216), + KQU( 3142372800384138465), KQU( 6201105606917248196), + KQU(18337516409359270184), KQU(15042268695665115339), + KQU(15188246541383283846), KQU(12800028693090114519), + KQU( 5992859621101493472), KQU(18278043971816803521), + KQU( 9002773075219424560), KQU( 7325707116943598353), + KQU( 7930571931248040822), KQU( 5645275869617023448), + KQU( 7266107455295958487), KQU( 4363664528273524411), + KQU(14313875763787479809), KQU(17059695613553486802), + KQU( 9247761425889940932), KQU(13704726459237593128), + KQU( 2701312427328909832), KQU(17235532008287243115), + KQU(14093147761491729538), KQU( 6247352273768386516), + KQU( 8268710048153268415), KQU( 7985295214477182083), + KQU(15624495190888896807), KQU( 3772753430045262788), + KQU( 9133991620474991698), KQU( 5665791943316256028), + KQU( 7551996832462193473), KQU(13163729206798953877), + KQU( 9263532074153846374), KQU( 1015460703698618353), + KQU(17929874696989519390), KQU(18257884721466153847), + KQU(16271867543011222991), KQU( 3905971519021791941), + KQU(16814488397137052085), KQU( 1321197685504621613), + KQU( 2870359191894002181), KQU(14317282970323395450), + KQU(13663920845511074366), KQU( 2052463995796539594), + KQU(14126345686431444337), KQU( 1727572121947022534), + KQU(17793552254485594241), KQU( 6738857418849205750), + KQU( 1282987123157442952), KQU(16655480021581159251), + KQU( 6784587032080183866), KQU(14726758805359965162), + KQU( 7577995933961987349), KQU(12539609320311114036), + KQU(10789773033385439494), KQU( 8517001497411158227), + KQU(10075543932136339710), KQU(14838152340938811081), + KQU( 9560840631794044194), KQU(17445736541454117475), + KQU(10633026464336393186), KQU(15705729708242246293), + KQU( 1117517596891411098), KQU( 4305657943415886942), + KQU( 4948856840533979263), KQU(16071681989041789593), + KQU(13723031429272486527), KQU( 7639567622306509462), + KQU(12670424537483090390), KQU( 9715223453097197134), + KQU( 5457173389992686394), KQU( 289857129276135145), + KQU(17048610270521972512), KQU( 692768013309835485), + KQU(14823232360546632057), KQU(18218002361317895936), + KQU( 3281724260212650204), KQU(16453957266549513795), + KQU( 8592711109774511881), KQU( 929825123473369579), + KQU(15966784769764367791), KQU( 9627344291450607588), + KQU(10849555504977813287), KQU( 9234566913936339275), + KQU( 6413807690366911210), KQU(10862389016184219267), + KQU(13842504799335374048), KQU( 1531994113376881174), + KQU( 2081314867544364459), KQU(16430628791616959932), + KQU( 8314714038654394368), KQU( 9155473892098431813), + KQU(12577843786670475704), KQU( 4399161106452401017), + KQU( 1668083091682623186), KQU( 1741383777203714216), + KQU( 2162597285417794374), KQU(15841980159165218736), + KQU( 1971354603551467079), KQU( 1206714764913205968), + KQU( 4790860439591272330), KQU(14699375615594055799), + KQU( 8374423871657449988), KQU(10950685736472937738), + KQU( 697344331343267176), KQU(10084998763118059810), + KQU(12897369539795983124), KQU(12351260292144383605), + KQU( 1268810970176811234), KQU( 7406287800414582768), + KQU( 516169557043807831), KQU( 5077568278710520380), + KQU( 3828791738309039304), KQU( 7721974069946943610), + KQU( 3534670260981096460), KQU( 4865792189600584891), + KQU(16892578493734337298), KQU( 9161499464278042590), + KQU(11976149624067055931), KQU(13219479887277343990), + KQU(14161556738111500680), KQU(14670715255011223056), + KQU( 4671205678403576558), KQU(12633022931454259781), + KQU(14821376219869187646), KQU( 751181776484317028), + KQU( 2192211308839047070), KQU(11787306362361245189), + KQU(10672375120744095707), KQU( 4601972328345244467), + KQU(15457217788831125879), KQU( 8464345256775460809), + KQU(10191938789487159478), KQU( 6184348739615197613), + KQU(11425436778806882100), KQU( 2739227089124319793), + KQU( 461464518456000551), KQU( 4689850170029177442), + KQU( 6120307814374078625), KQU(11153579230681708671), + KQU( 7891721473905347926), KQU(10281646937824872400), + KQU( 3026099648191332248), KQU( 8666750296953273818), + KQU(14978499698844363232), KQU(13303395102890132065), + KQU( 8182358205292864080), KQU(10560547713972971291), + KQU(11981635489418959093), KQU( 3134621354935288409), + KQU(11580681977404383968), KQU(14205530317404088650), + KQU( 5997789011854923157), KQU(13659151593432238041), + KQU(11664332114338865086), KQU( 7490351383220929386), + KQU( 7189290499881530378), KQU(15039262734271020220), + KQU( 2057217285976980055), KQU( 555570804905355739), + KQU(11235311968348555110), KQU(13824557146269603217), + KQU(16906788840653099693), KQU( 7222878245455661677), + KQU( 5245139444332423756), KQU( 4723748462805674292), + KQU(12216509815698568612), KQU(17402362976648951187), + KQU(17389614836810366768), KQU( 4880936484146667711), + KQU( 9085007839292639880), KQU(13837353458498535449), + KQU(11914419854360366677), KQU(16595890135313864103), + KQU( 6313969847197627222), KQU(18296909792163910431), + KQU(10041780113382084042), KQU( 2499478551172884794), + KQU(11057894246241189489), KQU( 9742243032389068555), + KQU(12838934582673196228), KQU(13437023235248490367), + KQU(13372420669446163240), KQU( 6752564244716909224), + KQU( 7157333073400313737), KQU(12230281516370654308), + KQU( 1182884552219419117), KQU( 2955125381312499218), + KQU(10308827097079443249), KQU( 1337648572986534958), + KQU(16378788590020343939), KQU( 108619126514420935), + KQU( 3990981009621629188), KQU( 5460953070230946410), + KQU( 9703328329366531883), KQU(13166631489188077236), + KQU( 1104768831213675170), KQU( 3447930458553877908), + KQU( 8067172487769945676), KQU( 5445802098190775347), + KQU( 3244840981648973873), KQU(17314668322981950060), + KQU( 5006812527827763807), KQU(18158695070225526260), + KQU( 2824536478852417853), KQU(13974775809127519886), + KQU( 9814362769074067392), KQU(17276205156374862128), + KQU(11361680725379306967), KQU( 3422581970382012542), + KQU(11003189603753241266), KQU(11194292945277862261), + KQU( 6839623313908521348), KQU(11935326462707324634), + KQU( 1611456788685878444), KQU(13112620989475558907), + KQU( 517659108904450427), KQU(13558114318574407624), + KQU(15699089742731633077), KQU( 4988979278862685458), + KQU( 8111373583056521297), KQU( 3891258746615399627), + KQU( 8137298251469718086), KQU(12748663295624701649), + KQU( 4389835683495292062), KQU( 5775217872128831729), + KQU( 9462091896405534927), KQU( 8498124108820263989), + KQU( 8059131278842839525), KQU(10503167994254090892), + KQU(11613153541070396656), KQU(18069248738504647790), + KQU( 570657419109768508), KQU( 3950574167771159665), + KQU( 5514655599604313077), KQU( 2908460854428484165), + KQU(10777722615935663114), KQU(12007363304839279486), + KQU( 9800646187569484767), KQU( 8795423564889864287), + KQU(14257396680131028419), KQU( 6405465117315096498), + KQU( 7939411072208774878), KQU(17577572378528990006), + KQU(14785873806715994850), KQU(16770572680854747390), + KQU(18127549474419396481), KQU(11637013449455757750), + KQU(14371851933996761086), KQU( 3601181063650110280), + KQU( 4126442845019316144), KQU(10198287239244320669), + KQU(18000169628555379659), KQU(18392482400739978269), + KQU( 6219919037686919957), KQU( 3610085377719446052), + KQU( 2513925039981776336), KQU(16679413537926716955), + KQU(12903302131714909434), KQU( 5581145789762985009), + KQU(12325955044293303233), KQU(17216111180742141204), + KQU( 6321919595276545740), KQU( 3507521147216174501), + KQU( 9659194593319481840), KQU(11473976005975358326), + KQU(14742730101435987026), KQU( 492845897709954780), + KQU(16976371186162599676), KQU(17712703422837648655), + KQU( 9881254778587061697), KQU( 8413223156302299551), + KQU( 1563841828254089168), KQU( 9996032758786671975), + KQU( 138877700583772667), KQU(13003043368574995989), + KQU( 4390573668650456587), KQU( 8610287390568126755), + KQU(15126904974266642199), KQU( 6703637238986057662), + KQU( 2873075592956810157), KQU( 6035080933946049418), + KQU(13382846581202353014), KQU( 7303971031814642463), + KQU(18418024405307444267), KQU( 5847096731675404647), + KQU( 4035880699639842500), KQU(11525348625112218478), + KQU( 3041162365459574102), KQU( 2604734487727986558), + KQU(15526341771636983145), KQU(14556052310697370254), + KQU(12997787077930808155), KQU( 9601806501755554499), + KQU(11349677952521423389), KQU(14956777807644899350), + KQU(16559736957742852721), KQU(12360828274778140726), + KQU( 6685373272009662513), KQU(16932258748055324130), + KQU(15918051131954158508), KQU( 1692312913140790144), + KQU( 546653826801637367), KQU( 5341587076045986652), + KQU(14975057236342585662), KQU(12374976357340622412), + KQU(10328833995181940552), KQU(12831807101710443149), + KQU(10548514914382545716), KQU( 2217806727199715993), + KQU(12627067369242845138), KQU( 4598965364035438158), + KQU( 150923352751318171), KQU(14274109544442257283), + KQU( 4696661475093863031), KQU( 1505764114384654516), + KQU(10699185831891495147), KQU( 2392353847713620519), + KQU( 3652870166711788383), KQU( 8640653276221911108), + KQU( 3894077592275889704), KQU( 4918592872135964845), + KQU(16379121273281400789), KQU(12058465483591683656), + KQU(11250106829302924945), KQU( 1147537556296983005), + KQU( 6376342756004613268), KQU(14967128191709280506), + KQU(18007449949790627628), KQU( 9497178279316537841), + KQU( 7920174844809394893), KQU(10037752595255719907), + KQU(15875342784985217697), KQU(15311615921712850696), + KQU( 9552902652110992950), KQU(14054979450099721140), + KQU( 5998709773566417349), KQU(18027910339276320187), + KQU( 8223099053868585554), KQU( 7842270354824999767), + KQU( 4896315688770080292), KQU(12969320296569787895), + KQU( 2674321489185759961), KQU( 4053615936864718439), + KQU(11349775270588617578), KQU( 4743019256284553975), + KQU( 5602100217469723769), KQU(14398995691411527813), + KQU( 7412170493796825470), KQU( 836262406131744846), + KQU( 8231086633845153022), KQU( 5161377920438552287), + KQU( 8828731196169924949), KQU(16211142246465502680), + KQU( 3307990879253687818), KQU( 5193405406899782022), + KQU( 8510842117467566693), KQU( 6070955181022405365), + KQU(14482950231361409799), KQU(12585159371331138077), + KQU( 3511537678933588148), KQU( 2041849474531116417), + KQU(10944936685095345792), KQU(18303116923079107729), + KQU( 2720566371239725320), KQU( 4958672473562397622), + KQU( 3032326668253243412), KQU(13689418691726908338), + KQU( 1895205511728843996), KQU( 8146303515271990527), + KQU(16507343500056113480), KQU( 473996939105902919), + KQU( 9897686885246881481), KQU(14606433762712790575), + KQU( 6732796251605566368), KQU( 1399778120855368916), + KQU( 935023885182833777), KQU(16066282816186753477), + KQU( 7291270991820612055), KQU(17530230393129853844), + KQU(10223493623477451366), KQU(15841725630495676683), + KQU(17379567246435515824), KQU( 8588251429375561971), + KQU(18339511210887206423), KQU(17349587430725976100), + KQU(12244876521394838088), KQU( 6382187714147161259), + KQU(12335807181848950831), KQU(16948885622305460665), + KQU(13755097796371520506), KQU(14806740373324947801), + KQU( 4828699633859287703), KQU( 8209879281452301604), + KQU(12435716669553736437), KQU(13970976859588452131), + KQU( 6233960842566773148), KQU(12507096267900505759), + KQU( 1198713114381279421), KQU(14989862731124149015), + KQU(15932189508707978949), KQU( 2526406641432708722), + KQU( 29187427817271982), KQU( 1499802773054556353), + KQU(10816638187021897173), KQU( 5436139270839738132), + KQU( 6659882287036010082), KQU( 2154048955317173697), + KQU(10887317019333757642), KQU(16281091802634424955), + KQU(10754549879915384901), KQU(10760611745769249815), + KQU( 2161505946972504002), KQU( 5243132808986265107), + KQU(10129852179873415416), KQU( 710339480008649081), + KQU( 7802129453068808528), KQU(17967213567178907213), + KQU(15730859124668605599), KQU(13058356168962376502), + KQU( 3701224985413645909), KQU(14464065869149109264), + KQU( 9959272418844311646), KQU(10157426099515958752), + KQU(14013736814538268528), KQU(17797456992065653951), + KQU(17418878140257344806), KQU(15457429073540561521), + KQU( 2184426881360949378), KQU( 2062193041154712416), + KQU( 8553463347406931661), KQU( 4913057625202871854), + KQU( 2668943682126618425), KQU(17064444737891172288), + KQU( 4997115903913298637), KQU(12019402608892327416), + KQU(17603584559765897352), KQU(11367529582073647975), + KQU( 8211476043518436050), KQU( 8676849804070323674), + KQU(18431829230394475730), KQU(10490177861361247904), + KQU( 9508720602025651349), KQU( 7409627448555722700), + KQU( 5804047018862729008), KQU(11943858176893142594), + KQU(11908095418933847092), KQU( 5415449345715887652), + KQU( 1554022699166156407), KQU( 9073322106406017161), + KQU( 7080630967969047082), KQU(18049736940860732943), + KQU(12748714242594196794), KQU( 1226992415735156741), + KQU(17900981019609531193), KQU(11720739744008710999), + KQU( 3006400683394775434), KQU(11347974011751996028), + KQU( 3316999628257954608), KQU( 8384484563557639101), + KQU(18117794685961729767), KQU( 1900145025596618194), + KQU(17459527840632892676), KQU( 5634784101865710994), + KQU( 7918619300292897158), KQU( 3146577625026301350), + KQU( 9955212856499068767), KQU( 1873995843681746975), + KQU( 1561487759967972194), KQU( 8322718804375878474), + KQU(11300284215327028366), KQU( 4667391032508998982), + KQU( 9820104494306625580), KQU(17922397968599970610), + KQU( 1784690461886786712), KQU(14940365084341346821), + KQU( 5348719575594186181), KQU(10720419084507855261), + KQU(14210394354145143274), KQU( 2426468692164000131), + KQU(16271062114607059202), KQU(14851904092357070247), + KQU( 6524493015693121897), KQU( 9825473835127138531), + KQU(14222500616268569578), KQU(15521484052007487468), + KQU(14462579404124614699), KQU(11012375590820665520), + KQU(11625327350536084927), KQU(14452017765243785417), + KQU( 9989342263518766305), KQU( 3640105471101803790), + KQU( 4749866455897513242), KQU(13963064946736312044), + KQU(10007416591973223791), KQU(18314132234717431115), + KQU( 3286596588617483450), KQU( 7726163455370818765), + KQU( 7575454721115379328), KQU( 5308331576437663422), + KQU(18288821894903530934), KQU( 8028405805410554106), + KQU(15744019832103296628), KQU( 149765559630932100), + KQU( 6137705557200071977), KQU(14513416315434803615), + KQU(11665702820128984473), KQU( 218926670505601386), + KQU( 6868675028717769519), KQU(15282016569441512302), + KQU( 5707000497782960236), KQU( 6671120586555079567), + KQU( 2194098052618985448), KQU(16849577895477330978), + KQU(12957148471017466283), KQU( 1997805535404859393), + KQU( 1180721060263860490), KQU(13206391310193756958), + KQU(12980208674461861797), KQU( 3825967775058875366), + KQU(17543433670782042631), KQU( 1518339070120322730), + KQU(16344584340890991669), KQU( 2611327165318529819), + KQU(11265022723283422529), KQU( 4001552800373196817), + KQU(14509595890079346161), KQU( 3528717165416234562), + KQU(18153222571501914072), KQU( 9387182977209744425), + KQU(10064342315985580021), KQU(11373678413215253977), + KQU( 2308457853228798099), KQU( 9729042942839545302), + KQU( 7833785471140127746), KQU( 6351049900319844436), + KQU(14454610627133496067), KQU(12533175683634819111), + KQU(15570163926716513029), KQU(13356980519185762498) }; TEST_BEGIN(test_gen_rand_32) diff --git a/test/unit/util.c b/test/unit/util.c index c11d5984..8ab39a45 100644 --- a/test/unit/util.c +++ b/test/unit/util.c @@ -52,8 +52,8 @@ TEST_BEGIN(test_malloc_strtoumax) const char *expected_errno_name; uintmax_t expected_x; }; -#define ERR(e) e, #e -#define UMAX(x) ((uintmax_t)x##ULL) +#define ERR(e) e, #e +#define KUMAX(x) ((uintmax_t)x##ULL) struct test_s tests[] = { {"0", "0", -1, ERR(EINVAL), UINTMAX_MAX}, {"0", "0", 1, ERR(EINVAL), UINTMAX_MAX}, @@ -64,51 +64,51 @@ TEST_BEGIN(test_malloc_strtoumax) {"++3", "++3", 0, ERR(EINVAL), UINTMAX_MAX}, {"-", "-", 0, ERR(EINVAL), UINTMAX_MAX}, - {"42", "", 0, ERR(0), UMAX(42)}, - {"+42", "", 0, ERR(0), UMAX(42)}, - {"-42", "", 0, ERR(0), UMAX(-42)}, - {"042", "", 0, ERR(0), UMAX(042)}, - {"+042", "", 0, ERR(0), UMAX(042)}, - {"-042", "", 0, ERR(0), UMAX(-042)}, - {"0x42", "", 0, ERR(0), UMAX(0x42)}, - {"+0x42", "", 0, ERR(0), UMAX(0x42)}, - {"-0x42", "", 0, ERR(0), UMAX(-0x42)}, + {"42", "", 0, ERR(0), KUMAX(42)}, + {"+42", "", 0, ERR(0), KUMAX(42)}, + {"-42", "", 0, ERR(0), KUMAX(-42)}, + {"042", "", 0, ERR(0), KUMAX(042)}, + {"+042", "", 0, ERR(0), KUMAX(042)}, + {"-042", "", 0, ERR(0), KUMAX(-042)}, + {"0x42", "", 0, ERR(0), KUMAX(0x42)}, + {"+0x42", "", 0, ERR(0), KUMAX(0x42)}, + {"-0x42", "", 0, ERR(0), KUMAX(-0x42)}, - {"0", "", 0, ERR(0), UMAX(0)}, - {"1", "", 0, ERR(0), UMAX(1)}, + {"0", "", 0, ERR(0), KUMAX(0)}, + {"1", "", 0, ERR(0), KUMAX(1)}, - {"42", "", 0, ERR(0), UMAX(42)}, - {" 42", "", 0, ERR(0), UMAX(42)}, - {"42 ", " ", 0, ERR(0), UMAX(42)}, - {"0x", "x", 0, ERR(0), UMAX(0)}, - {"42x", "x", 0, ERR(0), UMAX(42)}, + {"42", "", 0, ERR(0), KUMAX(42)}, + {" 42", "", 0, ERR(0), KUMAX(42)}, + {"42 ", " ", 0, ERR(0), KUMAX(42)}, + {"0x", "x", 0, ERR(0), KUMAX(0)}, + {"42x", "x", 0, ERR(0), KUMAX(42)}, - {"07", "", 0, ERR(0), UMAX(7)}, - {"010", "", 0, ERR(0), UMAX(8)}, - {"08", "8", 0, ERR(0), UMAX(0)}, - {"0_", "_", 0, ERR(0), UMAX(0)}, + {"07", "", 0, ERR(0), KUMAX(7)}, + {"010", "", 0, ERR(0), KUMAX(8)}, + {"08", "8", 0, ERR(0), KUMAX(0)}, + {"0_", "_", 0, ERR(0), KUMAX(0)}, - {"0x", "x", 0, ERR(0), UMAX(0)}, - {"0X", "X", 0, ERR(0), UMAX(0)}, - {"0xg", "xg", 0, ERR(0), UMAX(0)}, - {"0XA", "", 0, ERR(0), UMAX(10)}, + {"0x", "x", 0, ERR(0), KUMAX(0)}, + {"0X", "X", 0, ERR(0), KUMAX(0)}, + {"0xg", "xg", 0, ERR(0), KUMAX(0)}, + {"0XA", "", 0, ERR(0), KUMAX(10)}, - {"010", "", 10, ERR(0), UMAX(10)}, - {"0x3", "x3", 10, ERR(0), UMAX(0)}, + {"010", "", 10, ERR(0), KUMAX(10)}, + {"0x3", "x3", 10, ERR(0), KUMAX(0)}, - {"12", "2", 2, ERR(0), UMAX(1)}, - {"78", "8", 8, ERR(0), UMAX(7)}, - {"9a", "a", 10, ERR(0), UMAX(9)}, - {"9A", "A", 10, ERR(0), UMAX(9)}, - {"fg", "g", 16, ERR(0), UMAX(15)}, - {"FG", "G", 16, ERR(0), UMAX(15)}, - {"0xfg", "g", 16, ERR(0), UMAX(15)}, - {"0XFG", "G", 16, ERR(0), UMAX(15)}, - {"z_", "_", 36, ERR(0), UMAX(35)}, - {"Z_", "_", 36, ERR(0), UMAX(35)} + {"12", "2", 2, ERR(0), KUMAX(1)}, + {"78", "8", 8, ERR(0), KUMAX(7)}, + {"9a", "a", 10, ERR(0), KUMAX(9)}, + {"9A", "A", 10, ERR(0), KUMAX(9)}, + {"fg", "g", 16, ERR(0), KUMAX(15)}, + {"FG", "G", 16, ERR(0), KUMAX(15)}, + {"0xfg", "g", 16, ERR(0), KUMAX(15)}, + {"0XFG", "G", 16, ERR(0), KUMAX(15)}, + {"z_", "_", 36, ERR(0), KUMAX(35)}, + {"Z_", "_", 36, ERR(0), KUMAX(35)} }; #undef ERR -#undef UMAX +#undef KUMAX unsigned i; for (i = 0; i < sizeof(tests)/sizeof(struct test_s); i++) { From 999e1b5cc74e299a25cc718ddf9fae370cf45264 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 29 May 2014 09:03:00 +0900 Subject: [PATCH 044/352] Fix thd_join on win64 --- test/src/thd.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/src/thd.c b/test/src/thd.c index 7e53625f..c9d00658 100644 --- a/test/src/thd.c +++ b/test/src/thd.c @@ -14,8 +14,11 @@ void thd_join(thd_t thd, void **ret) { - if (WaitForSingleObject(thd, INFINITE) == WAIT_OBJECT_0 && ret) - GetExitCodeThread(thd, (LPDWORD) ret); + if (WaitForSingleObject(thd, INFINITE) == WAIT_OBJECT_0 && ret) { + DWORD exit_code; + GetExitCodeThread(thd, (LPDWORD) &exit_code); + *ret = (void *)(uintptr_t)exit_code; + } } #else From ff2e999667cbd06e5e80c243277c1f3c72d6d263 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 29 May 2014 16:33:02 +0900 Subject: [PATCH 045/352] Don't use msvc_compat's C99 headers with MSVC versions that have (some) C99 support --- configure.ac | 4 ++++ include/msvc_compat/{ => C99}/inttypes.h | 0 include/msvc_compat/{ => C99}/stdbool.h | 0 include/msvc_compat/{ => C99}/stdint.h | 0 4 files changed, 4 insertions(+) rename include/msvc_compat/{ => C99}/inttypes.h (100%) rename include/msvc_compat/{ => C99}/stdbool.h (100%) rename include/msvc_compat/{ => C99}/stdint.h (100%) diff --git a/configure.ac b/configure.ac index 58522499..5aeaa088 100644 --- a/configure.ac +++ b/configure.ac @@ -155,6 +155,10 @@ if test "x${ac_cv_big_endian}" = "x1" ; then AC_DEFINE_UNQUOTED([JEMALLOC_BIG_ENDIAN], [ ]) fi +if test "x${je_cv_msvc}" = "xyes" -a "x${ac_cv_header_inttypes_h}" = "xno"; then + CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat/C99" +fi + AC_CHECK_SIZEOF([void *]) if test "x${ac_cv_sizeof_void_p}" = "x8" ; then LG_SIZEOF_PTR=3 diff --git a/include/msvc_compat/inttypes.h b/include/msvc_compat/C99/inttypes.h similarity index 100% rename from include/msvc_compat/inttypes.h rename to include/msvc_compat/C99/inttypes.h diff --git a/include/msvc_compat/stdbool.h b/include/msvc_compat/C99/stdbool.h similarity index 100% rename from include/msvc_compat/stdbool.h rename to include/msvc_compat/C99/stdbool.h diff --git a/include/msvc_compat/stdint.h b/include/msvc_compat/C99/stdint.h similarity index 100% rename from include/msvc_compat/stdint.h rename to include/msvc_compat/C99/stdint.h From 8c6157558aca6cb764b4f312c3d4f285664ef3e7 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 29 May 2014 16:58:21 +0900 Subject: [PATCH 046/352] Add -FS flag to support parallel builds with MSVC 2013 --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 5aeaa088..045f62e2 100644 --- a/configure.ac +++ b/configure.ac @@ -141,6 +141,7 @@ if test "x$CFLAGS" = "x" ; then JE_CFLAGS_APPEND([-Zi]) JE_CFLAGS_APPEND([-MT]) JE_CFLAGS_APPEND([-W3]) + JE_CFLAGS_APPEND([-FS]) CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat" fi fi From 6f6704c35b28e919552a50e9e1d89a75a8b7c962 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 29 May 2014 17:01:10 +0900 Subject: [PATCH 047/352] Make in-tree MSVC builds work --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 045f62e2..4944c44a 100644 --- a/configure.ac +++ b/configure.ac @@ -142,7 +142,7 @@ if test "x$CFLAGS" = "x" ; then JE_CFLAGS_APPEND([-MT]) JE_CFLAGS_APPEND([-W3]) JE_CFLAGS_APPEND([-FS]) - CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat" + CPPFLAGS="$CPPFLAGS -I${srcdir}/include/msvc_compat" fi fi dnl Append EXTRA_CFLAGS to CFLAGS, if defined. @@ -157,7 +157,7 @@ if test "x${ac_cv_big_endian}" = "x1" ; then fi if test "x${je_cv_msvc}" = "xyes" -a "x${ac_cv_header_inttypes_h}" = "xno"; then - CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat/C99" + CPPFLAGS="$CPPFLAGS -I${srcdir}/include/msvc_compat/C99" fi AC_CHECK_SIZEOF([void *]) From 0b5c92213fbafc52c5b5a5dc84e91eacc812ae0b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 1 Jun 2014 22:05:08 -0700 Subject: [PATCH 048/352] Fix fallback lg_floor() implementations. --- include/jemalloc/internal/util.h | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index 78648238..54aed8ec 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -152,9 +152,9 @@ lg_floor(size_t x) { #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) - return ((8 << LG_SIZEOF_PTR - 1) - __builtin_clz(x)); + return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) - return ((8 << LG_SIZEOF_PTR - 1) - __builtin_clzl(x)); + return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clzl(x)); #else # error "Unsupported type sizes for lg_floor()" #endif @@ -164,16 +164,22 @@ JEMALLOC_INLINE size_t lg_floor(size_t x) { - x |= (x >> 1); - x |= (x >> 2); - x |= (x >> 4); - x |= (x >> 8); - x |= (x >> 16); + x |= (x >> 1); + x |= (x >> 2); + x |= (x >> 4); + x |= (x >> 8); + x |= (x >> 16); #if (LG_SIZEOF_PTR == 3 && LG_SIZEOF_PTR == LG_SIZEOF_LONG) - x |= (x >> 32); - return (65 - ffsl(~x)); + x |= (x >> 32); + if (x == KZU(0xffffffffffffffff)) + return (63); + x++; + return (ffsl(x) - 2); #elif (LG_SIZEOF_PTR == 2) - return (33 - ffs(~x)); + if (x == KZU(0xffffffff)) + return (31); + x++; + return (ffs(x) - 2); #else # error "Unsupported type sizes for lg_floor()" #endif From 9c3a10fdf6baa5ddb042b6adbef1ff1b3c613ce3 Mon Sep 17 00:00:00 2001 From: Richard Diamond Date: Wed, 28 May 2014 21:37:02 -0500 Subject: [PATCH 049/352] Try to use __builtin_ffsl if ffsl is unavailable. Some platforms (like those using Newlib) don't have ffs/ffsl. This commit adds a check to configure.ac for __builtin_ffsl if ffsl isn't found. __builtin_ffsl performs the same function as ffsl, and has the added benefit of being available on any platform utilizing Gcc-compatible compiler. This change does not address the used of ffs in the MALLOCX_ARENA() macro. --- configure.ac | 30 +++++++++++++++---- include/jemalloc/internal/arena.h | 2 +- include/jemalloc/internal/bitmap.h | 4 +-- .../jemalloc/internal/jemalloc_internal.h.in | 3 ++ .../internal/jemalloc_internal_decls.h | 10 ++++--- .../internal/jemalloc_internal_defs.h.in | 7 +++++ include/jemalloc/internal/util.h | 26 ++++++++++++++-- src/arena.c | 2 +- src/rtree.c | 4 +-- 9 files changed, 71 insertions(+), 17 deletions(-) diff --git a/configure.ac b/configure.ac index 4944c44a..3d36b5f8 100644 --- a/configure.ac +++ b/configure.ac @@ -1109,9 +1109,11 @@ elif test "x${force_tls}" = "x1" ; then fi dnl ============================================================================ -dnl Check for ffsl(3), and fail if not found. This function exists on all -dnl platforms that jemalloc currently has a chance of functioning on without -dnl modification. +dnl Check for ffsl(3), then __builtin_ffsl(), and fail if neither are found. +dnl One of those two functions should (theoretically) exist on all platforms +dnl that jemalloc currently has a chance of functioning on without modification. +dnl We additionally assume ffs() or __builtin_ffs() are defined if +dnl ffsl() or __builtin_ffsl() are defined, respectively. JE_COMPILABLE([a program using ffsl], [ #include #include @@ -1122,8 +1124,26 @@ JE_COMPILABLE([a program using ffsl], [ printf("%d\n", rv); } ], [je_cv_function_ffsl]) -if test "x${je_cv_function_ffsl}" != "xyes" ; then - AC_MSG_ERROR([Cannot build without ffsl(3)]) +if test "x${je_cv_function_ffsl}" == "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [ffsl]) + AC_DEFINE([JEMALLOC_INTERNAL_FFS], [ffs]) +else + JE_COMPILABLE([a program using __builtin_ffsl], [ + #include + #include + #include + ], [ + { + int rv = __builtin_ffsl(0x08); + printf("%d\n", rv); + } + ], [je_cv_gcc_builtin_ffsl]) + if test "x${je_cv_gcc_builtin_ffsl}" == "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [__builtin_ffsl]) + AC_DEFINE([JEMALLOC_INTERNAL_FFS], [__builtin_ffs]) + else + AC_MSG_ERROR([Cannot build without ffsl(3) or __builtin_ffsl()]) + fi fi dnl ============================================================================ diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 2dc9501d..cb73283b 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -970,7 +970,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) /* Rescale (factor powers of 2 out of the numerator and denominator). */ interval = bin_info->reg_interval; - shift = ffs(interval) - 1; + shift = jemalloc_ffs(interval) - 1; diff >>= shift; interval >>= shift; diff --git a/include/jemalloc/internal/bitmap.h b/include/jemalloc/internal/bitmap.h index 605ebac5..6db4ab70 100644 --- a/include/jemalloc/internal/bitmap.h +++ b/include/jemalloc/internal/bitmap.h @@ -130,11 +130,11 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) i = binfo->nlevels - 1; g = bitmap[binfo->levels[i].group_offset]; - bit = ffsl(g) - 1; + bit = jemalloc_ffsl(g) - 1; while (i > 0) { i--; g = bitmap[binfo->levels[i].group_offset + bit]; - bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffsl(g) - 1); + bit = (bit << LG_BITMAP_GROUP_NBITS) + (jemalloc_ffsl(g) - 1); } bitmap_set(bitmap, binfo, bit); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 491345c9..f2cd743f 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -215,6 +215,9 @@ static const bool config_ivsalloc = # ifdef __tile__ # define LG_QUANTUM 4 # endif +# ifdef __le32__ +# define LG_QUANTUM 4 +# endif # ifndef LG_QUANTUM # error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS" # endif diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h index 7775ab38..fa590404 100644 --- a/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -15,11 +15,13 @@ #else # include # include -# include -# if !defined(SYS_write) && defined(__NR_write) -# define SYS_write __NR_write +# if !defined(__pnacl__) && !defined(__native_client__) +# include +# if !defined(SYS_write) && defined(__NR_write) +# define SYS_write __NR_write +# endif +# include # endif -# include # include # include #endif diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index a9a50f14..65ac76c0 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -152,6 +152,13 @@ /* TLS is used to map arenas and magazine caches to threads. */ #undef JEMALLOC_TLS +/* + * ffs()/ffsl() functions to use for bitmapping. Don't use these directly; + * instead, use jemalloc_ffs() or jemalloc_ffsl() from util.h. + */ +#undef JEMALLOC_INTERNAL_FFSL +#undef JEMALLOC_INTERNAL_FFS + /* * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside * within jemalloc-owned chunks before dereferencing them. diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index 54aed8ec..d2b7a967 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -109,6 +109,8 @@ void malloc_printf(const char *format, ...) #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +int jemalloc_ffsl(long bitmap); +int jemalloc_ffs(int bitmap); size_t pow2_ceil(size_t x); size_t lg_floor(size_t x); void set_errno(int errnum); @@ -116,6 +118,26 @@ int get_errno(void); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) + +/* Sanity check: */ +#if !defined(JEMALLOC_INTERNAL_FFSL) || !defined(JEMALLOC_INTERNAL_FFS) +# error Both JEMALLOC_INTERNAL_FFSL && JEMALLOC_INTERNAL_FFS should have been defined by configure +#endif + +JEMALLOC_ALWAYS_INLINE int +jemalloc_ffsl(long bitmap) +{ + + return (JEMALLOC_INTERNAL_FFSL(bitmap)); +} + +JEMALLOC_ALWAYS_INLINE int +jemalloc_ffs(int bitmap) +{ + + return (JEMALLOC_INTERNAL_FFS(bitmap)); +} + /* Compute the smallest power of 2 that is >= x. */ JEMALLOC_INLINE size_t pow2_ceil(size_t x) @@ -174,12 +196,12 @@ lg_floor(size_t x) if (x == KZU(0xffffffffffffffff)) return (63); x++; - return (ffsl(x) - 2); + return (jemalloc_ffsl(x) - 2); #elif (LG_SIZEOF_PTR == 2) if (x == KZU(0xffffffff)) return (31); x++; - return (ffs(x) - 2); + return (jemalloc_ffs(x) - 2); #else # error "Unsupported type sizes for lg_floor()" #endif diff --git a/src/arena.c b/src/arena.c index c392419e..d3fe0fba 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2483,7 +2483,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) * be twice as large in order to maintain alignment. */ if (config_fill && opt_redzone) { - size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); + size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - 1); if (align_min <= REDZONE_MINSIZE) { bin_info->redzone_size = REDZONE_MINSIZE; pad_size = 0; diff --git a/src/rtree.c b/src/rtree.c index 205957ac..87b0b154 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -9,8 +9,8 @@ rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc) assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3)); - bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1; - bits_in_leaf = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1; + bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1; + bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1; if (bits > bits_in_leaf) { height = 1 + (bits - bits_in_leaf) / bits_per_level; if ((height-1) * bits_per_level + bits_in_leaf != bits) From 94ed6812bc04a6171d1a801f2740355f458d5c9c Mon Sep 17 00:00:00 2001 From: Richard Diamond Date: Wed, 28 May 2014 21:47:15 -0500 Subject: [PATCH 050/352] Don't catch fork()ing events for Native Client. Native Client doesn't allow forking, thus there is no need to catch fork()ing events for Native Client. Additionally, without this commit, jemalloc will introduce an unresolved pthread_atfork() in PNaCl Rust bins. --- src/jemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 43a494e4..0983c00d 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -792,7 +792,7 @@ malloc_init_hard(void) ncpus = malloc_ncpus(); #if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ - && !defined(_WIN32)) + && !defined(_WIN32) && !defined(__native_client__)) /* LinuxThreads's pthread_atfork() allocates. */ if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, jemalloc_postfork_child) != 0) { From 3e310b34eb53eb331981ecda2ea5f10cf6956747 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Wed, 28 May 2014 19:04:06 -0700 Subject: [PATCH 051/352] Fix -Wsign-compare warnings --- src/prof.c | 4 ++-- src/util.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/prof.c b/src/prof.c index b64386e3..0eb7dbdb 100644 --- a/src/prof.c +++ b/src/prof.c @@ -1093,7 +1093,7 @@ label_open_close_error: #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) #define VSEQ_INVALID UINT64_C(0xffffffffffffffff) static void -prof_dump_filename(char *filename, char v, int64_t vseq) +prof_dump_filename(char *filename, char v, uint64_t vseq) { cassert(config_prof); @@ -1101,7 +1101,7 @@ prof_dump_filename(char *filename, char v, int64_t vseq) if (vseq != VSEQ_INVALID) { /* "...v.heap" */ malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, - "%s.%d.%"PRIu64".%c%"PRId64".heap", + "%s.%d.%"PRIu64".%c%"PRIu64".heap", opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq); } else { /* "....heap" */ diff --git a/src/util.c b/src/util.c index 93a19fd1..9076be94 100644 --- a/src/util.c +++ b/src/util.c @@ -100,7 +100,7 @@ uintmax_t malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { uintmax_t ret, digit; - int b; + unsigned b; bool neg; const char *p, *ns; @@ -548,7 +548,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) assert(len == '?' || len == 'l'); assert_not_implemented(len != 'l'); s = va_arg(ap, char *); - slen = (prec < 0) ? strlen(s) : prec; + slen = (prec < 0) ? strlen(s) : (size_t)prec; APPEND_PADDED_S(s, slen, width, left_justify); f++; break; From 70807bc54b06bb259b6607541af44bc73a890bf6 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Wed, 28 May 2014 19:04:33 -0700 Subject: [PATCH 052/352] Fix -Wsometimes-uninitialized warnings --- src/util.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util.c b/src/util.c index 9076be94..1717f08e 100644 --- a/src/util.c +++ b/src/util.c @@ -381,7 +381,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) case 'p': /* Synthetic; used for %p. */ \ val = va_arg(ap, uintptr_t); \ break; \ - default: not_reached(); \ + default: \ + not_reached(); \ + val = 0; \ } \ } while (0) From 994fad9bdaaa18273f2089856c2637cfb0c307bd Mon Sep 17 00:00:00 2001 From: Richard Diamond Date: Tue, 3 Jun 2014 02:39:18 -0500 Subject: [PATCH 053/352] Add check for madvise(2) to configure.ac. Some platforms, such as Google's Portable Native Client, use Newlib and thus lack access to madvise(2). In those instances, pages_purge() is transformed into a no-op. --- configure.ac | 14 ++++++++++++++ .../jemalloc/internal/jemalloc_internal_defs.h.in | 5 +++++ src/chunk_mmap.c | 7 +++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 3d36b5f8..29edcb6a 100644 --- a/configure.ac +++ b/configure.ac @@ -1191,6 +1191,20 @@ if test "x${je_cv_osatomic}" = "xyes" ; then AC_DEFINE([JEMALLOC_OSATOMIC], [ ]) fi +dnl ============================================================================ +dnl Check for madvise(2). + +JE_COMPILABLE([madvise(2)], [ +#include +], [ + { + madvise((void *)0, 0, 0); + } +], [je_cv_madvise]) +if test "x${je_cv_madvise}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_MADVISE], [ ]) +fi + dnl ============================================================================ dnl Check whether __sync_{add,sub}_and_fetch() are available despite dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined. diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 65ac76c0..93716b0a 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -52,6 +52,11 @@ */ #undef JEMALLOC_HAVE_BUILTIN_CLZ +/* + * Defined if madvise(2) is available. + */ +#undef JEMALLOC_HAVE_MADVISE + /* * Defined if OSSpin*() functions are available, as provided by Darwin, and * documented in the spinlock(3) manual page. diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c index f960e068..65137b41 100644 --- a/src/chunk_mmap.c +++ b/src/chunk_mmap.c @@ -121,7 +121,7 @@ pages_purge(void *addr, size_t length) #ifdef _WIN32 VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE); unzeroed = true; -#else +#elif defined(JEMALLOC_HAVE_MADVISE) # ifdef JEMALLOC_PURGE_MADVISE_DONTNEED # define JEMALLOC_MADV_PURGE MADV_DONTNEED # define JEMALLOC_MADV_ZEROS true @@ -129,12 +129,15 @@ pages_purge(void *addr, size_t length) # define JEMALLOC_MADV_PURGE MADV_FREE # define JEMALLOC_MADV_ZEROS false # else -# error "No method defined for purging unused dirty pages." +# error "No madvise(2) flag defined for purging unused dirty pages." # endif int err = madvise(addr, length, JEMALLOC_MADV_PURGE); unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0); # undef JEMALLOC_MADV_PURGE # undef JEMALLOC_MADV_ZEROS +#else + /* Last resort no-op. */ + unzeroed = true; #endif return (unzeroed); } From 1a3eafd1b045163f27e4a5acf01280edfe28c309 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 4 Jun 2014 12:09:08 +0900 Subject: [PATCH 054/352] Check for __builtin_ffsl before ffsl. When building with -O0, GCC doesn't use builtins for ffs and ffsl calls, and uses library function calls instead. But the Android NDK doesn't have those functions exported from any library, leading to build failure. However, using __builtin_ffs* uses the builtin inlines. --- configure.ac | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/configure.ac b/configure.ac index 29edcb6a..f456bd21 100644 --- a/configure.ac +++ b/configure.ac @@ -1109,43 +1109,44 @@ elif test "x${force_tls}" = "x1" ; then fi dnl ============================================================================ -dnl Check for ffsl(3), then __builtin_ffsl(), and fail if neither are found. +dnl Check for __builtin_ffsl(), then ffsl(3), and fail if neither are found. dnl One of those two functions should (theoretically) exist on all platforms dnl that jemalloc currently has a chance of functioning on without modification. dnl We additionally assume ffs() or __builtin_ffs() are defined if dnl ffsl() or __builtin_ffsl() are defined, respectively. -JE_COMPILABLE([a program using ffsl], [ +JE_COMPILABLE([a program using __builtin_ffsl], [ #include #include #include ], [ { - int rv = ffsl(0x08); + int rv = __builtin_ffsl(0x08); printf("%d\n", rv); } -], [je_cv_function_ffsl]) -if test "x${je_cv_function_ffsl}" == "xyes" ; then - AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [ffsl]) - AC_DEFINE([JEMALLOC_INTERNAL_FFS], [ffs]) +], [je_cv_gcc_builtin_ffsl]) +if test "x${je_cv_gcc_builtin_ffsl}" == "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [__builtin_ffsl]) + AC_DEFINE([JEMALLOC_INTERNAL_FFS], [__builtin_ffs]) else - JE_COMPILABLE([a program using __builtin_ffsl], [ + JE_COMPILABLE([a program using ffsl], [ #include #include #include ], [ { - int rv = __builtin_ffsl(0x08); + int rv = ffsl(0x08); printf("%d\n", rv); } - ], [je_cv_gcc_builtin_ffsl]) - if test "x${je_cv_gcc_builtin_ffsl}" == "xyes" ; then - AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [__builtin_ffsl]) - AC_DEFINE([JEMALLOC_INTERNAL_FFS], [__builtin_ffs]) + ], [je_cv_function_ffsl]) + if test "x${je_cv_function_ffsl}" == "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [ffsl]) + AC_DEFINE([JEMALLOC_INTERNAL_FFS], [ffs]) else AC_MSG_ERROR([Cannot build without ffsl(3) or __builtin_ffsl()]) fi fi + dnl ============================================================================ dnl Check for atomic(9) operations as provided on FreeBSD. From 8f50ec8eda262e87ad547ec50b6ca928ea3e31c4 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 4 Jun 2014 12:12:55 +0900 Subject: [PATCH 055/352] Use JEMALLOC_INTERNAL_FFSL in STATIC_PAGE_SHIFT test --- configure.ac | 79 ++++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/configure.ac b/configure.ac index f456bd21..e9775342 100644 --- a/configure.ac +++ b/configure.ac @@ -935,6 +935,44 @@ if test "x$enable_xmalloc" = "x1" ; then fi AC_SUBST([enable_xmalloc]) +dnl ============================================================================ +dnl Check for __builtin_ffsl(), then ffsl(3), and fail if neither are found. +dnl One of those two functions should (theoretically) exist on all platforms +dnl that jemalloc currently has a chance of functioning on without modification. +dnl We additionally assume ffs() or __builtin_ffs() are defined if +dnl ffsl() or __builtin_ffsl() are defined, respectively. +JE_COMPILABLE([a program using __builtin_ffsl], [ +#include +#include +#include +], [ + { + int rv = __builtin_ffsl(0x08); + printf("%d\n", rv); + } +], [je_cv_gcc_builtin_ffsl]) +if test "x${je_cv_gcc_builtin_ffsl}" == "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [__builtin_ffsl]) + AC_DEFINE([JEMALLOC_INTERNAL_FFS], [__builtin_ffs]) +else + JE_COMPILABLE([a program using ffsl], [ + #include + #include + #include + ], [ + { + int rv = ffsl(0x08); + printf("%d\n", rv); + } + ], [je_cv_function_ffsl]) + if test "x${je_cv_function_ffsl}" == "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [ffsl]) + AC_DEFINE([JEMALLOC_INTERNAL_FFS], [ffs]) + else + AC_MSG_ERROR([Cannot build without ffsl(3) or __builtin_ffsl()]) + fi +fi + AC_CACHE_CHECK([STATIC_PAGE_SHIFT], [je_cv_static_page_shift], AC_RUN_IFELSE([AC_LANG_PROGRAM( @@ -961,7 +999,7 @@ AC_CACHE_CHECK([STATIC_PAGE_SHIFT], if (result == -1) { return 1; } - result = ffsl(result) - 1; + result = JEMALLOC_INTERNAL_FFSL(result) - 1; f = fopen("conftest.out", "w"); if (f == NULL) { @@ -1108,45 +1146,6 @@ elif test "x${force_tls}" = "x1" ; then AC_MSG_ERROR([Failed to configure TLS, which is mandatory for correct function]) fi -dnl ============================================================================ -dnl Check for __builtin_ffsl(), then ffsl(3), and fail if neither are found. -dnl One of those two functions should (theoretically) exist on all platforms -dnl that jemalloc currently has a chance of functioning on without modification. -dnl We additionally assume ffs() or __builtin_ffs() are defined if -dnl ffsl() or __builtin_ffsl() are defined, respectively. -JE_COMPILABLE([a program using __builtin_ffsl], [ -#include -#include -#include -], [ - { - int rv = __builtin_ffsl(0x08); - printf("%d\n", rv); - } -], [je_cv_gcc_builtin_ffsl]) -if test "x${je_cv_gcc_builtin_ffsl}" == "xyes" ; then - AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [__builtin_ffsl]) - AC_DEFINE([JEMALLOC_INTERNAL_FFS], [__builtin_ffs]) -else - JE_COMPILABLE([a program using ffsl], [ - #include - #include - #include - ], [ - { - int rv = ffsl(0x08); - printf("%d\n", rv); - } - ], [je_cv_function_ffsl]) - if test "x${je_cv_function_ffsl}" == "xyes" ; then - AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [ffsl]) - AC_DEFINE([JEMALLOC_INTERNAL_FFS], [ffs]) - else - AC_MSG_ERROR([Cannot build without ffsl(3) or __builtin_ffsl()]) - fi -fi - - dnl ============================================================================ dnl Check for atomic(9) operations as provided on FreeBSD. From 5921ba7b0c3b3278c54d569dee37deab2768b70b Mon Sep 17 00:00:00 2001 From: Valerii Hiora Date: Fri, 16 May 2014 16:28:20 +0300 Subject: [PATCH 056/352] Support for iOS compilation --- config.sub | 2 ++ configure.ac | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config.sub b/config.sub index 61cb4bc2..c4cc9836 100755 --- a/config.sub +++ b/config.sub @@ -1400,6 +1400,8 @@ case $os in -mac*) os=`echo $os | sed -e 's|mac|macos|'` ;; + -ios*) + ;; -linux-dietlibc) os=-linux-dietlibc ;; diff --git a/configure.ac b/configure.ac index e9775342..48863a59 100644 --- a/configure.ac +++ b/configure.ac @@ -264,7 +264,7 @@ dnl definitions need to be seen before any headers are included, which is a pain dnl to make happen otherwise. default_munmap="1" case "${host}" in - *-*-darwin*) + *-*-darwin* | *-*-ios*) CFLAGS="$CFLAGS" abi="macho" AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) From 6f533c1903a1d067dacfca2f06c6cc9754fdf67e Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 10 Jun 2014 18:18:22 +0900 Subject: [PATCH 057/352] Ensure the default purgeable zone is after the default zone on OS X --- src/zone.c | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/zone.c b/src/zone.c index e0302ef4..a722287b 100644 --- a/src/zone.c +++ b/src/zone.c @@ -176,6 +176,7 @@ register_zone(void) * register jemalloc's. */ malloc_zone_t *default_zone = malloc_default_zone(); + malloc_zone_t *purgeable_zone = NULL; if (!default_zone->zone_name || strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) { return; @@ -237,22 +238,37 @@ register_zone(void) * run time. */ if (malloc_default_purgeable_zone != NULL) - malloc_default_purgeable_zone(); + purgeable_zone = malloc_default_purgeable_zone(); /* Register the custom zone. At this point it won't be the default. */ malloc_zone_register(&zone); - /* - * Unregister and reregister the default zone. On OSX >= 10.6, - * unregistering takes the last registered zone and places it at the - * location of the specified zone. Unregistering the default zone thus - * makes the last registered one the default. On OSX < 10.6, - * unregistering shifts all registered zones. The first registered zone - * then becomes the default. - */ do { default_zone = malloc_default_zone(); + /* + * Unregister and reregister the default zone. On OSX >= 10.6, + * unregistering takes the last registered zone and places it + * at the location of the specified zone. Unregistering the + * default zone thus makes the last registered one the default. + * On OSX < 10.6, unregistering shifts all registered zones. + * The first registered zone then becomes the default. + */ malloc_zone_unregister(default_zone); malloc_zone_register(default_zone); + /* + * On OSX 10.6, having the default purgeable zone appear before + * the default zone makes some things crash because it thinks it + * owns the default zone allocated pointers. We thus unregister/ + * re-register it in order to ensure it's always after the + * default zone. On OSX < 10.6, there is no purgeable zone, so + * this does nothing. On OSX >= 10.6, unregistering replaces the + * purgeable zone with the last registered zone above, i.e the + * default zone. Registering it again then puts it at the end, + * obviously after the default zone. + */ + if (purgeable_zone) { + malloc_zone_unregister(purgeable_zone); + malloc_zone_register(purgeable_zone); + } } while (malloc_default_zone() != &zone); } From c521df5dcf7410898cabdcb556f919535cf16d19 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 12 Jun 2014 13:07:31 +0900 Subject: [PATCH 058/352] Allow to build with clang-cl --- include/msvc_compat/C99/stdbool.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/msvc_compat/C99/stdbool.h b/include/msvc_compat/C99/stdbool.h index da9ee8b8..d92160eb 100644 --- a/include/msvc_compat/C99/stdbool.h +++ b/include/msvc_compat/C99/stdbool.h @@ -5,7 +5,11 @@ /* MSVC doesn't define _Bool or bool in C, but does have BOOL */ /* Note this doesn't pass autoconf's test because (bool) 0.5 != true */ +/* Clang-cl uses MSVC headers, so needs msvc_compat, but has _Bool as + * a built-in type. */ +#ifndef __clang__ typedef BOOL _Bool; +#endif #define bool _Bool #define true 1 From 79230fef31428a133683c236bedcc1560f8fcfd8 Mon Sep 17 00:00:00 2001 From: Steven Stewart-Gallus Date: Thu, 19 Jun 2014 16:11:43 -0700 Subject: [PATCH 059/352] Fix unportable == operator in configure scripts Now this code is more portable and now people can use faster shells than Bash such as Dash. To use a faster shell with autoconf set the CONFIG_SHELL environment variable to the shell and run the configure script with the shell. --- configure.ac | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index 48863a59..645bd466 100644 --- a/configure.ac +++ b/configure.ac @@ -833,7 +833,7 @@ have_dss="1" dnl Check whether the BSD/SUSv1 sbrk() exists. If not, disable DSS support. AC_CHECK_FUNC([sbrk], [have_sbrk="1"], [have_sbrk="0"]) if test "x$have_sbrk" = "x1" ; then - if test "x$sbrk_deprecated" == "x1" ; then + if test "x$sbrk_deprecated" = "x1" ; then AC_MSG_RESULT([Disabling dss allocation because sbrk is deprecated]) have_dss="0" fi @@ -951,7 +951,7 @@ JE_COMPILABLE([a program using __builtin_ffsl], [ printf("%d\n", rv); } ], [je_cv_gcc_builtin_ffsl]) -if test "x${je_cv_gcc_builtin_ffsl}" == "xyes" ; then +if test "x${je_cv_gcc_builtin_ffsl}" = "xyes" ; then AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [__builtin_ffsl]) AC_DEFINE([JEMALLOC_INTERNAL_FFS], [__builtin_ffs]) else @@ -965,7 +965,7 @@ else printf("%d\n", rv); } ], [je_cv_function_ffsl]) - if test "x${je_cv_function_ffsl}" == "xyes" ; then + if test "x${je_cv_function_ffsl}" = "xyes" ; then AC_DEFINE([JEMALLOC_INTERNAL_FFSL], [ffsl]) AC_DEFINE([JEMALLOC_INTERNAL_FFS], [ffs]) else From ffa259841c6a4b6dae4ed74f02bb38703e190065 Mon Sep 17 00:00:00 2001 From: "Manuel A. Fernandez Montecelo" Date: Tue, 29 Jul 2014 23:11:26 +0100 Subject: [PATCH 060/352] Add OpenRISC/or1k LG_QUANTUM size definition --- include/jemalloc/internal/jemalloc_internal.h.in | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index f2cd743f..1c2f3d44 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -203,6 +203,9 @@ static const bool config_ivsalloc = # ifdef __mips__ # define LG_QUANTUM 3 # endif +# ifdef __or1k__ +# define LG_QUANTUM 3 +# endif # ifdef __powerpc__ # define LG_QUANTUM 4 # endif From b433d7a87b27ff1e4ccea5103bc0a95afbf58ea4 Mon Sep 17 00:00:00 2001 From: "Manuel A. Fernandez Montecelo" Date: Tue, 29 Jul 2014 23:15:26 +0100 Subject: [PATCH 061/352] Update config.{guess,sub} to more recent versions, to add better support to OpenRISC/or1k (among others) --- config.guess | 192 ++++++++------------------------------------------- config.sub | 21 +++--- 2 files changed, 37 insertions(+), 176 deletions(-) diff --git a/config.guess b/config.guess index b79252d6..1f5c50c0 100755 --- a/config.guess +++ b/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2013 Free Software Foundation, Inc. +# Copyright 1992-2014 Free Software Foundation, Inc. -timestamp='2013-06-10' +timestamp='2014-03-23' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2013 Free Software Foundation, Inc. +Copyright 1992-2014 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -149,7 +149,7 @@ Linux|GNU|GNU/*) LIBC=gnu #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` ;; esac @@ -826,7 +826,7 @@ EOF *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 exit ;; - i*:MSYS*:*) + *:MSYS*:*) echo ${UNAME_MACHINE}-pc-msys exit ;; i*:windows32*:*) @@ -969,10 +969,10 @@ EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } ;; - or1k:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + openrisc*:Linux:*:*) + echo or1k-unknown-linux-${LIBC} exit ;; - or32:Linux:*:*) + or32:Linux:*:* | or1k*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-${LIBC} exit ;; padre:Linux:*:*) @@ -1260,16 +1260,26 @@ EOF if test "$UNAME_PROCESSOR" = unknown ; then UNAME_PROCESSOR=powerpc fi - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then - if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null - then - case $UNAME_PROCESSOR in - i386) UNAME_PROCESSOR=x86_64 ;; - powerpc) UNAME_PROCESSOR=powerpc64 ;; - esac + if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi fi + elif test "$UNAME_PROCESSOR" = i386 ; then + # Avoid executing cc on OS X 10.9, as it ships with a stub + # that puts up a graphical alert prompting to install + # developer tools. Any system running Mac OS X 10.7 or + # later (Darwin 11 and later) is required to have a 64-bit + # processor. This is not true of the ARM version of Darwin + # that Apple uses in portable devices. + UNAME_PROCESSOR=x86_64 fi echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} exit ;; @@ -1361,154 +1371,6 @@ EOF exit ;; esac -eval $set_cc_for_build -cat >$dummy.c < -# include -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix\n"); exit (0); -#endif - -#if defined (hp300) && !defined (hpux) - printf ("m68k-hp-bsd\n"); exit (0); -#endif - -#if defined (NeXT) -#if !defined (__ARCHITECTURE__) -#define __ARCHITECTURE__ "m68k" -#endif - int version; - version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); - -#endif - -#if defined (vax) -# if !defined (ultrix) -# include -# if defined (BSD) -# if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -# else -# if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# endif -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# else - printf ("vax-dec-ultrix\n"); exit (0); -# endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} -EOF - -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && - { echo "$SYSTEM_NAME"; exit; } - -# Apollos put the system type in the environment. - -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } - -# Convex versions that predate uname can use getsysinfo(1) - -if [ -x /usr/convex/getsysinfo ] -then - case `getsysinfo -f cpu_type` in - c1*) - echo c1-convex-bsd - exit ;; - c2*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit ;; - c34*) - echo c34-convex-bsd - exit ;; - c38*) - echo c38-convex-bsd - exit ;; - c4*) - echo c4-convex-bsd - exit ;; - esac -fi - cat >&2 <." version="\ GNU config.sub ($timestamp) -Copyright 1992-2013 Free Software Foundation, Inc. +Copyright 1992-2014 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -283,8 +283,10 @@ case $basic_machine in | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ | mipsisa64 | mipsisa64el \ | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ | mipsisa64sb1 | mipsisa64sb1el \ | mipsisa64sr71k | mipsisa64sr71kel \ | mipsr5900 | mipsr5900el \ @@ -296,8 +298,7 @@ case $basic_machine in | nds32 | nds32le | nds32be \ | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ - | open8 \ - | or1k | or32 \ + | open8 | or1k | or1knd | or32 \ | pdp10 | pdp11 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ | pyramid \ @@ -402,8 +403,10 @@ case $basic_machine in | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa32r6-* | mipsisa32r6el-* \ | mipsisa64-* | mipsisa64el-* \ | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64r6-* | mipsisa64r6el-* \ | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ | mipsr5900-* | mipsr5900el-* \ @@ -415,6 +418,7 @@ case $basic_machine in | nios-* | nios2-* | nios2eb-* | nios2el-* \ | none-* | np1-* | ns16k-* | ns32k-* \ | open8-* \ + | or1k*-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ @@ -1376,7 +1380,7 @@ case $os in | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1400,8 +1404,6 @@ case $os in -mac*) os=`echo $os | sed -e 's|mac|macos|'` ;; - -ios*) - ;; -linux-dietlibc) os=-linux-dietlibc ;; @@ -1596,9 +1598,6 @@ case $basic_machine in mips*-*) os=-elf ;; - or1k-*) - os=-elf - ;; or32-*) os=-coff ;; From 1aa25a3ca28d8da347dc115636073493db791183 Mon Sep 17 00:00:00 2001 From: Michael Neumann Date: Tue, 5 Aug 2014 03:06:02 +0200 Subject: [PATCH 062/352] Support DragonFlyBSD Note that in contrast to FreeBSD, DragonFly does not work with force_lazy_lock enabled. --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 645bd466..83c60ec5 100644 --- a/configure.ac +++ b/configure.ac @@ -283,6 +283,11 @@ case "${host}" in AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) force_lazy_lock="1" ;; + *-*-dragonfly*) + CFLAGS="$CFLAGS" + abi="elf" + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + ;; *-*-linux*) CFLAGS="$CFLAGS" CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" From cf6032d0efbc2e3e9f736a8cd69846cf7427640b Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 30 Jul 2014 18:16:13 +0900 Subject: [PATCH 063/352] Remove ${srcroot} from cfghdrs_in, cfgoutputs_in and cfghdrs_tup in configure On Windows, srcroot may start with "drive:", which confuses autoconf's AC_CONFIG_* macros. The macros works equally well without ${srcroot}, provided some adjustment to Makefile.in. --- Makefile.in | 4 ++-- configure.ac | 46 +++++++++++++++++++++++----------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Makefile.in b/Makefile.in index 839bb08f..a21acd45 100644 --- a/Makefile.in +++ b/Makefile.in @@ -42,9 +42,9 @@ XSLTPROC := @XSLTPROC@ AUTOCONF := @AUTOCONF@ _RPATH = @RPATH@ RPATH = $(if $(1),$(call _RPATH,$(1))) -cfghdrs_in := @cfghdrs_in@ +cfghdrs_in := $(addprefix $(srcroot),@cfghdrs_in@) cfghdrs_out := @cfghdrs_out@ -cfgoutputs_in := @cfgoutputs_in@ +cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@) cfgoutputs_out := @cfgoutputs_out@ enable_autogen := @enable_autogen@ enable_code_coverage := @enable_code_coverage@ diff --git a/configure.ac b/configure.ac index 645bd466..bc3464fd 100644 --- a/configure.ac +++ b/configure.ac @@ -534,15 +534,15 @@ dnl jemalloc_protos_jet.h easy. je_="je_" AC_SUBST([je_]) -cfgoutputs_in="${srcroot}Makefile.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/html.xsl.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/manpages.xsl.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/jemalloc.xml.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_macros.h.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc_protos.h.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal.h.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/test.sh.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/include/test/jemalloc_test.h.in" +cfgoutputs_in="Makefile.in" +cfgoutputs_in="${cfgoutputs_in} doc/html.xsl.in" +cfgoutputs_in="${cfgoutputs_in} doc/manpages.xsl.in" +cfgoutputs_in="${cfgoutputs_in} doc/jemalloc.xml.in" +cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_macros.h.in" +cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_protos.h.in" +cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_internal.h.in" +cfgoutputs_in="${cfgoutputs_in} test/test.sh.in" +cfgoutputs_in="${cfgoutputs_in} test/include/test/jemalloc_test.h.in" cfgoutputs_out="Makefile" cfgoutputs_out="${cfgoutputs_out} doc/html.xsl" @@ -564,18 +564,18 @@ cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in" cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in" -cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_namespace.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_unnamespace.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/private_symbols.txt" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_namespace.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/public_unnamespace.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/size_classes.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_rename.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc_mangle.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/jemalloc.sh" -cfghdrs_in="${cfghdrs_in} ${srcroot}test/include/test/jemalloc_test_defs.h.in" +cfghdrs_in="include/jemalloc/jemalloc_defs.h.in" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_unnamespace.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.txt" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_unnamespace.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/size_classes.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_rename.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_mangle.sh" +cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc.sh" +cfghdrs_in="${cfghdrs_in} test/include/test/jemalloc_test_defs.h.in" cfghdrs_out="include/jemalloc/jemalloc_defs.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc${install_suffix}.h" @@ -593,8 +593,8 @@ cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/jemalloc_internal_defs.h" cfghdrs_out="${cfghdrs_out} test/include/test/jemalloc_test_defs.h" cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.in" -cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:${srcroot}include/jemalloc/internal/jemalloc_internal_defs.h.in" -cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:${srcroot}test/include/test/jemalloc_test_defs.h.in" +cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:include/jemalloc/internal/jemalloc_internal_defs.h.in" +cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:test/include/test/jemalloc_test_defs.h.in" dnl Silence irrelevant compiler warnings by default. AC_ARG_ENABLE([cc-silence], From 55c9aa10386b21af92f323d04bddc15691d48756 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Wed, 6 Aug 2014 16:10:08 -0700 Subject: [PATCH 064/352] Fix the bug that causes not allocating free run with lowest address. --- src/arena.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/arena.c b/src/arena.c index d3fe0fba..db699161 100644 --- a/src/arena.c +++ b/src/arena.c @@ -101,14 +101,18 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) uintptr_t a_mapelm = (uintptr_t)a; uintptr_t b_mapelm = (uintptr_t)b; - if (a_mapelm & CHUNK_MAP_KEY) + if (a_mapelm & CHUNK_MAP_KEY) a_size = a_mapelm & ~PAGE_MASK; else a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; ret = (a_size > b_size) - (a_size < b_size); - if (ret == 0 && (!(a_mapelm & CHUNK_MAP_KEY))) - ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); + if (ret == 0) { + if (!(a_mapelm & CHUNK_MAP_KEY)) + ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); + else + ret = -1; + } return (ret); } From ea73eb8f3e029f0a5697e78c6771b49063cf4138 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Wed, 6 Aug 2014 16:43:01 -0700 Subject: [PATCH 065/352] Reintroduce the comment that was removed in f9ff603. --- src/arena.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index db699161..118700b9 100644 --- a/src/arena.c +++ b/src/arena.c @@ -110,8 +110,12 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) if (ret == 0) { if (!(a_mapelm & CHUNK_MAP_KEY)) ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); - else + else { + /* + * Treat keys as if they are lower than anything else. + */ ret = -1; + } } return (ret); From a2ea54c98640eafc5bb256fa4369d5553499ac81 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 6 Aug 2014 23:36:19 -0700 Subject: [PATCH 066/352] Add atomic operations tests and fix latent bugs. --- Makefile.in | 3 +- include/jemalloc/internal/atomic.h | 41 +++++++++---- test/unit/atomic.c | 97 ++++++++++++++++++++++++++++++ 3 files changed, 128 insertions(+), 13 deletions(-) create mode 100644 test/unit/atomic.c diff --git a/Makefile.in b/Makefile.in index a21acd45..dfafe455 100644 --- a/Makefile.in +++ b/Makefile.in @@ -110,7 +110,8 @@ C_TESTLIB_SRCS := $(srcroot)test/src/math.c $(srcroot)test/src/mtx.c \ $(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \ $(srcroot)test/src/thd.c C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c -TESTS_UNIT := $(srcroot)test/unit/bitmap.c \ +TESTS_UNIT := $(srcroot)test/unit/atomic.c \ + $(srcroot)test/unit/bitmap.c \ $(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/hash.c \ $(srcroot)test/unit/junk.c \ diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h index 11a7b47f..a0488157 100644 --- a/include/jemalloc/internal/atomic.h +++ b/include/jemalloc/internal/atomic.h @@ -18,6 +18,17 @@ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES +/* + * All functions return the arithmetic result of the atomic operation. Some + * atomic operation APIs return the value prior to mutation, in which case the + * following functions must redundantly compute the result so that it can be + * returned. These functions are normally inlined, so the extra operations can + * be optimized away if the return values aren't used by the callers. + * + * atomic_add_( *p, x) { return (*p + x); } + * atomic_sub_( *p, x) { return (*p - x); } + */ + #ifndef JEMALLOC_ENABLE_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); @@ -52,14 +63,14 @@ JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { - return (InterlockedExchangeAdd64(p, x)); + return (InterlockedExchangeAdd64(p, x) + x); } JEMALLOC_INLINE uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x) { - return (InterlockedExchangeAdd64(p, -((int64_t)x))); + return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x); } #elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint64_t @@ -79,28 +90,31 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { + uint64_t t = x; asm volatile ( "lock; xaddq %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ + : "+r" (t), "=m" (*p) /* Outputs. */ : "m" (*p) /* Inputs. */ ); - return (x); + return (t + x); } JEMALLOC_INLINE uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x) { + uint64_t t; x = (uint64_t)(-(int64_t)x); + t = x; asm volatile ( "lock; xaddq %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ + : "+r" (t), "=m" (*p) /* Outputs. */ : "m" (*p) /* Inputs. */ ); - return (x); + return (t + x); } # elif (defined(JEMALLOC_ATOMIC9)) JEMALLOC_INLINE uint64_t @@ -164,14 +178,14 @@ JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) { - return (InterlockedExchangeAdd(p, x)); + return (InterlockedExchangeAdd(p, x) + x); } JEMALLOC_INLINE uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x) { - return (InterlockedExchangeAdd(p, -((int32_t)x))); + return (InterlockedExchangeAdd(p, -((int32_t)x)) - x); } #elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint32_t @@ -191,28 +205,31 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) { + uint32_t t = x; asm volatile ( "lock; xaddl %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ + : "+r" (t), "=m" (*p) /* Outputs. */ : "m" (*p) /* Inputs. */ ); - return (x); + return (t + x); } JEMALLOC_INLINE uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x) { + uint32_t t; x = (uint32_t)(-(int32_t)x); + t = x; asm volatile ( "lock; xaddl %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ + : "+r" (t), "=m" (*p) /* Outputs. */ : "m" (*p) /* Inputs. */ ); - return (x); + return (t + x); } #elif (defined(JEMALLOC_ATOMIC9)) JEMALLOC_INLINE uint32_t diff --git a/test/unit/atomic.c b/test/unit/atomic.c new file mode 100644 index 00000000..eb6136c7 --- /dev/null +++ b/test/unit/atomic.c @@ -0,0 +1,97 @@ +#include "test/jemalloc_test.h" + +#define TEST_STRUCT(p, t) \ +struct p##_test_s { \ + t accum0; \ + t x; \ +}; \ +typedef struct p##_test_s p##_test_t; + +#define TEST_BODY(p, t, PRI) do { \ + const p##_test_t tests[] = { \ + {-1, -1}, \ + {-1, 0}, \ + {-1, 1}, \ + \ + { 0, -1}, \ + { 0, 0}, \ + { 0, 1}, \ + \ + { 1, -1}, \ + { 1, 0}, \ + { 1, 1}, \ + \ + {0, -(1 << 22)}, \ + {0, (1 << 22)}, \ + {(1 << 22), -(1 << 22)}, \ + {(1 << 22), (1 << 22)} \ + }; \ + unsigned i; \ + \ + for (i = 0; i < sizeof(tests)/sizeof(p##_test_t); i++) { \ + t accum = tests[i].accum0; \ + assert_u64_eq(atomic_read_##p(&accum), tests[i].accum0, \ + "i=%u", i); \ + assert_u64_eq(atomic_add_##p(&accum, tests[i].x), \ + tests[i].accum0 + tests[i].x, \ + "i=%u, accum=%#"PRI", x=%#"PRI, \ + i, tests[i].accum0, tests[i].x); \ + assert_u64_eq(atomic_read_##p(&accum), accum, \ + "i=%u", i); \ + \ + accum = tests[i].accum0; \ + assert_u64_eq(atomic_sub_##p(&accum, tests[i].x), \ + tests[i].accum0 - tests[i].x, \ + "i=%u, accum=%#"PRI", x=%#"PRI, \ + i, tests[i].accum0, tests[i].x); \ + assert_u64_eq(atomic_read_##p(&accum), accum, \ + "i=%u", i); \ + } \ +} while (0) + +TEST_STRUCT(uint64, uint64_t) +TEST_BEGIN(test_atomic_uint64) +{ + +#if !(LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) + test_skip("64-bit atomic operations not supported"); +#else + TEST_BODY(uint64, uint64_t, PRIx64); +#endif +} +TEST_END + +TEST_STRUCT(uint32, uint32_t) +TEST_BEGIN(test_atomic_uint32) +{ + + TEST_BODY(uint32, uint32_t, PRIx32); +} +TEST_END + +TEST_STRUCT(z, size_t) +TEST_BEGIN(test_atomic_z) +{ + + TEST_BODY(z, size_t, "zx"); +} +TEST_END + +TEST_STRUCT(u, unsigned) +TEST_BEGIN(test_atomic_u) +{ + + TEST_BODY(u, unsigned, "x"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_atomic_uint64, + test_atomic_uint32, + test_atomic_z, + test_atomic_u)); +} From 1522937e9cbcfa24c881dc439cc454f9a34a7e88 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 6 Aug 2014 23:38:39 -0700 Subject: [PATCH 067/352] Fix the cactive statistic. Fix the cactive statistic to decrease (rather than increase) when active memory decreases. This regression was introduced by aa5113b1fdafd1129c22512837c6c3d66c295fc8 (Refactor overly large/complex functions) and first released in 3.5.0. --- src/arena.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/arena.c b/src/arena.c index 118700b9..c0ec98a8 100644 --- a/src/arena.c +++ b/src/arena.c @@ -382,9 +382,9 @@ arena_cactive_update(arena_t *arena, size_t add_pages, size_t sub_pages) { if (config_stats) { - ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + - add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - - sub_pages) << LG_PAGE); + ssize_t cactive_diff = CHUNK_CEILING((arena->nactive + add_pages + - sub_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << + LG_PAGE); if (cactive_diff != 0) stats_cactive_add(cactive_diff); } From 011dde96c52e37e897526e242e9e3018caafb751 Mon Sep 17 00:00:00 2001 From: Psi Mankoski Date: Mon, 11 Aug 2014 17:08:25 -0700 Subject: [PATCH 068/352] Set VERSION also when the source directory is a git submodule using a ".git" file pointing to the repo. directory. --- configure.ac | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 9c365eda..ede5f70f 100644 --- a/configure.ac +++ b/configure.ac @@ -1029,8 +1029,8 @@ dnl ============================================================================ dnl jemalloc configuration. dnl -dnl Set VERSION if source directory has an embedded git repository. -if test -d "${srcroot}.git" ; then +dnl Set VERSION if source directory has an embedded git repository or is a git submodule. +if test -e "${srcroot}.git" ; then git describe --long --abbrev=40 > ${srcroot}VERSION fi jemalloc_version=`cat ${srcroot}VERSION` From 04d60a132beed9e8c33f73b94fb9251b919073c8 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Fri, 18 Jul 2014 14:21:17 -0700 Subject: [PATCH 069/352] Maintain all the dirty runs in a linked list for each arena --- include/jemalloc/internal/arena.h | 6 ++++ src/arena.c | 47 +++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index cb73283b..3422f362 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -89,6 +89,9 @@ struct arena_chunk_map_s { }; /* union { ... }; */ #endif + /* Linkage for list of dirty runs. */ + ql_elm(arena_chunk_map_t) dr_link; + /* * Run address (or size) and various flags are stored together. The bit * layout looks like (assuming 32-bit system): @@ -333,6 +336,9 @@ struct arena_s { /* Tree of dirty-page-containing chunks this arena manages. */ arena_chunk_tree_t chunks_dirty; + /* List of dirty runs this arena manages. */ + arena_chunk_mapelms_t runs_dirty; + /* * In order to avoid rapid chunk allocation/deallocation when an arena * oscillates right on the cusp of needing a new chunk, cache the most diff --git a/src/arena.c b/src/arena.c index c0ec98a8..33977311 100644 --- a/src/arena.c +++ b/src/arena.c @@ -394,6 +394,7 @@ static void arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, size_t flag_dirty, size_t need_pages) { + arena_chunk_map_t *mapelm; size_t total_pages, rem_pages; total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> @@ -404,6 +405,11 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, rem_pages = total_pages - need_pages; arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); + if (flag_dirty != 0) { + /* If the run is dirty, it must be in the dirty list. */ + mapelm = arena_mapp_get(chunk, run_ind); + ql_remove(&arena->runs_dirty, mapelm, dr_link); + } arena_cactive_update(arena, need_pages, 0); arena->nactive += need_pages; @@ -416,6 +422,14 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, (rem_pages << LG_PAGE), flag_dirty); + mapelm = arena_mapp_get(chunk, run_ind+need_pages); + /* + * Append the trailing run at the end of the dirty list. + * We could also insert the run at the original place. + * Let us consider this later. + */ + ql_elm_new(mapelm, dr_link); + ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); } else { arena_mapbits_unallocated_set(chunk, run_ind+need_pages, (rem_pages << LG_PAGE), @@ -701,6 +715,11 @@ arena_chunk_alloc(arena_t *arena) /* Insert the run into the runs_avail tree. */ arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, false, false); + if (arena_mapbits_dirty_get(chunk, map_bias) != 0) { + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias); + ql_elm_new(mapelm, dr_link); + ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); + } return (chunk); } @@ -739,6 +758,7 @@ arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size) static void arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) { + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == @@ -754,6 +774,10 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) */ arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, false, false); + if (arena_mapbits_dirty_get(chunk, map_bias) != 0) { + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias); + ql_remove(&arena->runs_dirty, mapelm, dr_link); + } if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; @@ -1216,6 +1240,13 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, false, true); + /* If the successor is dirty, remove it from runs_dirty. */ + if (flag_dirty != 0) { + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, + run_ind+run_pages); + ql_remove(&arena->runs_dirty, mapelm, dr_link); + } + size += nrun_size; run_pages += nrun_pages; @@ -1244,6 +1275,13 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, arena_avail_remove(arena, chunk, run_ind, prun_pages, true, false); + /* If the predecessor is dirty, remove it from runs_dirty. */ + if (flag_dirty != 0) { + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, + run_ind); + ql_remove(&arena->runs_dirty, mapelm, dr_link); + } + size += prun_size; run_pages += prun_pages; @@ -1261,6 +1299,7 @@ static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) { arena_chunk_t *chunk; + arena_chunk_map_t *mapelm; size_t size, run_ind, run_pages, flag_dirty; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); @@ -1315,6 +1354,13 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); + if (dirty) { + /* Insert into runs_dirty list. */ + mapelm = arena_mapp_get(chunk, run_ind); + ql_elm_new(mapelm, dr_link); + ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); + } + /* Deallocate chunk if it is now completely unused. */ if (size == arena_maxclass) { assert(run_ind == map_bias); @@ -2437,6 +2483,7 @@ arena_new(arena_t *arena, unsigned ind) /* Initialize chunks. */ arena_chunk_dirty_new(&arena->chunks_dirty); + ql_new(&arena->runs_dirty); arena->spare = NULL; arena->nactive = 0; From a244e5078e8505978b5f63cfe6dcb3c9d63d2cb5 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Mon, 21 Jul 2014 10:23:36 -0700 Subject: [PATCH 070/352] Add dirty page counting for debug --- src/arena.c | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/arena.c b/src/arena.c index 33977311..3cf1abf6 100644 --- a/src/arena.c +++ b/src/arena.c @@ -923,11 +923,33 @@ arena_maybe_purge(arena_t *arena) static arena_chunk_t * chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) { - size_t *ndirty = (size_t *)arg; + size_t *ndirty = (size_t *)arg; - assert(chunk->ndirty != 0); - *ndirty += chunk->ndirty; - return (NULL); + assert(chunk->ndirty != 0); + *ndirty += chunk->ndirty; + return (NULL); +} + +static size_t +arena_dirty_count(arena_t *arena) +{ + size_t ndirty = 0; + arena_chunk_map_t *mapelm; + arena_chunk_t *chunk; + size_t pageind, npages; + + ql_foreach(mapelm, &arena->runs_dirty, dr_link) { + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); + pageind = arena_mapelm_to_pageind(mapelm); + assert(arena_mapbits_allocated_get(chunk, pageind) == 0); + assert(arena_mapbits_large_get(chunk, pageind) == 0); + assert(arena_mapbits_dirty_get(chunk, pageind) != 0); + npages = arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE; + ndirty += npages; + } + + return (ndirty); } static size_t @@ -1134,6 +1156,9 @@ arena_purge(arena_t *arena, bool all) arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, chunks_dirty_iter_cb, (void *)&ndirty); assert(ndirty == arena->ndirty); + + ndirty = arena_dirty_count(arena); + assert(ndirty == arena->ndirty); } assert(arena->ndirty > arena->npurgatory || all); assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - From e970800c780df918b80f8b914eeac475dd5f1ec4 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Mon, 21 Jul 2014 18:09:04 -0700 Subject: [PATCH 071/352] Purge dirty pages from the beginning of the dirty list. --- src/arena.c | 227 +++++++++++++++------------------------------------- 1 file changed, 66 insertions(+), 161 deletions(-) diff --git a/src/arena.c b/src/arena.c index 3cf1abf6..a78a66f6 100644 --- a/src/arena.c +++ b/src/arena.c @@ -973,86 +973,73 @@ arena_compute_npurgatory(arena_t *arena, bool all) return (npurgatory); } -static void -arena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, +static size_t +arena_stash_dirty(arena_t *arena, bool all, size_t npurgatory, arena_chunk_mapelms_t *mapelms) { - size_t pageind, npages; + arena_chunk_map_t *mapelm; + size_t nstashed = 0; + arena_chunk_t *chunk; + size_t pageind, npages, run_size; + arena_run_t *run; - /* - * Temporarily allocate free dirty runs within chunk. If all is false, - * only operate on dirty runs that are fragments; otherwise operate on - * all dirty runs. - */ - for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); - if (arena_mapbits_allocated_get(chunk, pageind) == 0) { - size_t run_size = - arena_mapbits_unallocated_size_get(chunk, pageind); + /* Add at least npurgatory pages to purge_list. */ + for (mapelm = ql_first(&arena->runs_dirty); mapelm != NULL; + mapelm = ql_first(&arena->runs_dirty)) { + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); + pageind = arena_mapelm_to_pageind(mapelm); + run_size = arena_mapbits_unallocated_size_get(chunk, pageind); + npages = run_size >> LG_PAGE; + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << + LG_PAGE)); - npages = run_size >> LG_PAGE; - assert(pageind + npages <= chunk_npages); - assert(arena_mapbits_dirty_get(chunk, pageind) == - arena_mapbits_dirty_get(chunk, pageind+npages-1)); + assert(pageind + npages <= chunk_npages); + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, pageind+npages-1)); - if (arena_mapbits_dirty_get(chunk, pageind) != 0 && - (all || arena_avail_adjac(chunk, pageind, - npages))) { - arena_run_t *run = (arena_run_t *)((uintptr_t) - chunk + (uintptr_t)(pageind << LG_PAGE)); + /* Temporarily allocate the free dirty run. */ + arena_run_split_large(arena, run, run_size, false); + /* Append to purge_list for later processing. */ + ql_elm_new(mapelm, dr_link); + ql_tail_insert(mapelms, mapelm, dr_link); - arena_run_split_large(arena, run, run_size, - false); - /* Append to list for later processing. */ - ql_elm_new(mapelm, u.ql_link); - ql_tail_insert(mapelms, mapelm, u.ql_link); - } - } else { - /* Skip run. */ - if (arena_mapbits_large_get(chunk, pageind) != 0) { - npages = arena_mapbits_large_size_get(chunk, - pageind) >> LG_PAGE; - } else { - size_t binind; - arena_bin_info_t *bin_info; - arena_run_t *run = (arena_run_t *)((uintptr_t) - chunk + (uintptr_t)(pageind << LG_PAGE)); + nstashed += npages; - assert(arena_mapbits_small_runind_get(chunk, - pageind) == 0); - binind = arena_bin_index(arena, run->bin); - bin_info = &arena_bin_info[binind]; - npages = bin_info->run_size >> LG_PAGE; - } - } + if (all == false && nstashed >= npurgatory) + break; } - assert(pageind == chunk_npages); - assert(chunk->ndirty == 0 || all == false); - assert(chunk->nruns_adjac == 0); + + return (nstashed); } static size_t -arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, - arena_chunk_mapelms_t *mapelms) +arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms) { - size_t npurged, pageind, npages, nmadvise; + size_t npurged, nmadvise; arena_chunk_map_t *mapelm; + arena_chunk_t *chunk; + size_t pageind, npages, run_size; - malloc_mutex_unlock(&arena->lock); if (config_stats) nmadvise = 0; npurged = 0; - ql_foreach(mapelm, mapelms, u.ql_link) { + + malloc_mutex_unlock(&arena->lock); + + ql_foreach(mapelm, mapelms, dr_link) { bool unzeroed; size_t flag_unzeroed, i; + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); pageind = arena_mapelm_to_pageind(mapelm); - npages = arena_mapbits_large_size_get(chunk, pageind) >> - LG_PAGE; + run_size = arena_mapbits_large_size_get(chunk, pageind); + npages = run_size >> LG_PAGE; + assert(pageind + npages <= chunk_npages); unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << - LG_PAGE)), (npages << LG_PAGE)); + LG_PAGE)), run_size); flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; + /* * Set the unzeroed flag for all pages, now that pages_purge() * has returned whether the pages were zeroed as a side effect @@ -1067,89 +1054,48 @@ arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, arena_mapbits_unzeroed_set(chunk, pageind+i, flag_unzeroed); } + npurged += npages; if (config_stats) nmadvise++; } + malloc_mutex_lock(&arena->lock); - if (config_stats) + + if (config_stats) { arena->stats.nmadvise += nmadvise; + arena->stats.purged += npurged; + } return (npurged); } static void -arena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, - arena_chunk_mapelms_t *mapelms) +arena_unstash_purged(arena_t *arena, arena_chunk_mapelms_t *mapelms) { arena_chunk_map_t *mapelm; + arena_chunk_t *chunk; + arena_run_t *run; size_t pageind; /* Deallocate runs. */ for (mapelm = ql_first(mapelms); mapelm != NULL; mapelm = ql_first(mapelms)) { - arena_run_t *run; - + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); pageind = arena_mapelm_to_pageind(mapelm); run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << LG_PAGE)); - ql_remove(mapelms, mapelm, u.ql_link); + ql_remove(mapelms, mapelm, dr_link); arena_run_dalloc(arena, run, false, true); } } -static inline size_t -arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) -{ - size_t npurged; - arena_chunk_mapelms_t mapelms; - - ql_new(&mapelms); - - /* - * If chunk is the spare, temporarily re-allocate it, 1) so that its - * run is reinserted into runs_avail, and 2) so that it cannot be - * completely discarded by another thread while arena->lock is dropped - * by this thread. Note that the arena_run_dalloc() call will - * implicitly deallocate the chunk, so no explicit action is required - * in this function to deallocate the chunk. - * - * Note that once a chunk contains dirty pages, it cannot again contain - * a single run unless 1) it is a dirty run, or 2) this function purges - * dirty pages and causes the transition to a single clean run. Thus - * (chunk == arena->spare) is possible, but it is not possible for - * this function to be called on the spare unless it contains a dirty - * run. - */ - if (chunk == arena->spare) { - assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); - assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); - - arena_chunk_alloc(arena); - } - - if (config_stats) - arena->stats.purged += chunk->ndirty; - - /* - * Operate on all dirty runs if there is no clean/dirty run - * fragmentation. - */ - if (chunk->nruns_adjac == 0) - all = true; - - arena_chunk_stash_dirty(arena, chunk, all, &mapelms); - npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); - arena_chunk_unstash_purged(arena, chunk, &mapelms); - - return (npurged); -} - -static void +void arena_purge(arena_t *arena, bool all) { - arena_chunk_t *chunk; - size_t npurgatory; + size_t npurgatory, npurgeable, npurged; + arena_chunk_mapelms_t purge_list; + if (config_debug) { size_t ndirty = 0; @@ -1175,58 +1121,17 @@ arena_purge(arena_t *arena, bool all) npurgatory = arena_compute_npurgatory(arena, all); arena->npurgatory += npurgatory; - while (npurgatory > 0) { - size_t npurgeable, npurged, nunpurged; + ql_new(&purge_list); - /* Get next chunk with dirty pages. */ - chunk = arena_chunk_dirty_first(&arena->chunks_dirty); - if (chunk == NULL) { - /* - * This thread was unable to purge as many pages as - * originally intended, due to races with other threads - * that either did some of the purging work, or re-used - * dirty pages. - */ - arena->npurgatory -= npurgatory; - return; - } - npurgeable = chunk->ndirty; - assert(npurgeable != 0); + npurgeable = arena_stash_dirty(arena, all, npurgatory, &purge_list); + assert(npurgeable >= npurgatory); + /* Actually we no longer need arena->npurgatory. */ + arena->npurgatory -= npurgatory; - if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { - /* - * This thread will purge all the dirty pages in chunk, - * so set npurgatory to reflect this thread's intent to - * purge the pages. This tends to reduce the chances - * of the following scenario: - * - * 1) This thread sets arena->npurgatory such that - * (arena->ndirty - arena->npurgatory) is at the - * threshold. - * 2) This thread drops arena->lock. - * 3) Another thread causes one or more pages to be - * dirtied, and immediately determines that it must - * purge dirty pages. - * - * If this scenario *does* play out, that's okay, - * because all of the purging work being done really - * needs to happen. - */ - arena->npurgatory += npurgeable - npurgatory; - npurgatory = npurgeable; - } + npurged = arena_purge_stashed(arena, &purge_list); + assert(npurged == npurgeable); - /* - * Keep track of how many pages are purgeable, versus how many - * actually get purged, and adjust counters accordingly. - */ - arena->npurgatory -= npurgeable; - npurgatory -= npurgeable; - npurged = arena_chunk_purge(arena, chunk, all); - nunpurged = npurgeable - npurged; - arena->npurgatory += nunpurged; - npurgatory += nunpurged; - } + arena_unstash_purged(arena, &purge_list); } void From 90737fcda150a5da3f4db1c3144ea24eed8de55b Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Mon, 21 Jul 2014 19:39:20 -0700 Subject: [PATCH 072/352] Remove chunks_dirty tree, nruns_avail and nruns_adjac since we no longer need to maintain the tree for dirty page purging. --- include/jemalloc/internal/arena.h | 19 --- src/arena.c | 187 ++---------------------------- 2 files changed, 10 insertions(+), 196 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 3422f362..f87dfe4d 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -167,24 +167,9 @@ struct arena_chunk_s { /* Arena that owns the chunk. */ arena_t *arena; - /* Linkage for tree of arena chunks that contain dirty runs. */ - rb_node(arena_chunk_t) dirty_link; - /* Number of dirty pages. */ size_t ndirty; - /* Number of available runs. */ - size_t nruns_avail; - - /* - * Number of available run adjacencies that purging could coalesce. - * Clean and dirty available runs are not coalesced, which causes - * virtual memory fragmentation. The ratio of - * (nruns_avail-nruns_adjac):nruns_adjac is used for tracking this - * fragmentation. - */ - size_t nruns_adjac; - /* * Map of pages within chunk that keeps track of free/large/small. The * first map_bias entries are omitted, since the chunk header does not @@ -193,7 +178,6 @@ struct arena_chunk_s { */ arena_chunk_map_t map[1]; /* Dynamically sized. */ }; -typedef rb_tree(arena_chunk_t) arena_chunk_tree_t; struct arena_run_s { /* Bin this run is associated with. */ @@ -333,9 +317,6 @@ struct arena_s { dss_prec_t dss_prec; - /* Tree of dirty-page-containing chunks this arena manages. */ - arena_chunk_tree_t chunks_dirty; - /* List of dirty runs this arena manages. */ arena_chunk_mapelms_t runs_dirty; diff --git a/src/arena.c b/src/arena.c index a78a66f6..24ed2ba2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -125,143 +125,18 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, u.rb_link, arena_avail_comp) -static inline int -arena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) -{ - - assert(a != NULL); - assert(b != NULL); - - /* - * Short-circuit for self comparison. The following comparison code - * would come to the same result, but at the cost of executing the slow - * path. - */ - if (a == b) - return (0); - - /* - * Order such that chunks with higher fragmentation are "less than" - * those with lower fragmentation -- purging order is from "least" to - * "greatest". Fragmentation is measured as: - * - * mean current avail run size - * -------------------------------- - * mean defragmented avail run size - * - * navail - * ----------- - * nruns_avail nruns_avail-nruns_adjac - * = ========================= = ----------------------- - * navail nruns_avail - * ----------------------- - * nruns_avail-nruns_adjac - * - * The following code multiplies away the denominator prior to - * comparison, in order to avoid division. - * - */ - { - size_t a_val = (a->nruns_avail - a->nruns_adjac) * - b->nruns_avail; - size_t b_val = (b->nruns_avail - b->nruns_adjac) * - a->nruns_avail; - - if (a_val < b_val) - return (1); - if (a_val > b_val) - return (-1); - } - /* - * Break ties by chunk address. For fragmented chunks, report lower - * addresses as "lower", so that fragmentation reduction happens first - * at lower addresses. However, use the opposite ordering for - * unfragmented chunks, in order to increase the chances of - * re-allocating dirty runs. - */ - { - uintptr_t a_chunk = (uintptr_t)a; - uintptr_t b_chunk = (uintptr_t)b; - int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); - if (a->nruns_adjac == 0) { - assert(b->nruns_adjac == 0); - ret = -ret; - } - return (ret); - } -} - -/* Generate red-black tree functions. */ -rb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, - dirty_link, arena_chunk_dirty_comp) - -static inline bool -arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) -{ - bool ret; - - if (pageind-1 < map_bias) - ret = false; - else { - ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); - assert(ret == false || arena_mapbits_dirty_get(chunk, - pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); - } - return (ret); -} - -static inline bool -arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) -{ - bool ret; - - if (pageind+npages == chunk_npages) - ret = false; - else { - assert(pageind+npages < chunk_npages); - ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); - assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) - != arena_mapbits_dirty_get(chunk, pageind+npages)); - } - return (ret); -} - -static inline bool -arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) -{ - - return (arena_avail_adjac_pred(chunk, pageind) || - arena_avail_adjac_succ(chunk, pageind, npages)); -} - static void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, - size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) + size_t npages) { assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); - /* - * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be - * removed and reinserted even if the run to be inserted is clean. - */ - if (chunk->ndirty != 0) - arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); - - if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) - chunk->nruns_adjac++; - if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) - chunk->nruns_adjac++; - chunk->nruns_avail++; - assert(chunk->nruns_avail > chunk->nruns_adjac); - if (arena_mapbits_dirty_get(chunk, pageind) != 0) { arena->ndirty += npages; chunk->ndirty += npages; } - if (chunk->ndirty != 0) - arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, pageind)); @@ -269,33 +144,16 @@ arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, static void arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, - size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) + size_t npages) { assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); - /* - * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be - * removed and reinserted even if the run to be removed is clean. - */ - if (chunk->ndirty != 0) - arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); - - if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) - chunk->nruns_adjac--; - if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) - chunk->nruns_adjac--; - chunk->nruns_avail--; - assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail - == 0 && chunk->nruns_adjac == 0)); - if (arena_mapbits_dirty_get(chunk, pageind) != 0) { arena->ndirty -= npages; chunk->ndirty -= npages; } - if (chunk->ndirty != 0) - arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, pageind)); @@ -404,7 +262,7 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, assert(need_pages <= total_pages); rem_pages = total_pages - need_pages; - arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); + arena_avail_remove(arena, chunk, run_ind, total_pages); if (flag_dirty != 0) { /* If the run is dirty, it must be in the dirty list. */ mapelm = arena_mapp_get(chunk, run_ind); @@ -440,8 +298,7 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1)); } - arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, - false, true); + arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages); } } @@ -660,9 +517,6 @@ arena_chunk_init_hard(arena_t *arena) */ chunk->ndirty = 0; - chunk->nruns_avail = 0; - chunk->nruns_adjac = 0; - /* * Initialize the map to contain one maximal free untouched run. Mark * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. @@ -713,8 +567,7 @@ arena_chunk_alloc(arena_t *arena) } /* Insert the run into the runs_avail tree. */ - arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, - false, false); + arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); if (arena_mapbits_dirty_get(chunk, map_bias) != 0) { arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias); ql_elm_new(mapelm, dr_link); @@ -772,8 +625,7 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) * Remove run from the runs_avail tree, so that the arena does not use * it. */ - arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, - false, false); + arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); if (arena_mapbits_dirty_get(chunk, map_bias) != 0) { arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias); ql_remove(&arena->runs_dirty, mapelm, dr_link); @@ -920,16 +772,6 @@ arena_maybe_purge(arena_t *arena) arena_purge(arena, false); } -static arena_chunk_t * -chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) -{ - size_t *ndirty = (size_t *)arg; - - assert(chunk->ndirty != 0); - *ndirty += chunk->ndirty; - return (NULL); -} - static size_t arena_dirty_count(arena_t *arena) { @@ -1097,13 +939,7 @@ arena_purge(arena_t *arena, bool all) arena_chunk_mapelms_t purge_list; if (config_debug) { - size_t ndirty = 0; - - arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, - chunks_dirty_iter_cb, (void *)&ndirty); - assert(ndirty == arena->ndirty); - - ndirty = arena_dirty_count(arena); + size_t ndirty = arena_dirty_count(arena); assert(ndirty == arena->ndirty); } assert(arena->ndirty > arena->npurgatory || all); @@ -1167,8 +1003,7 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, run_ind+run_pages+nrun_pages-1) == nrun_size); assert(arena_mapbits_dirty_get(chunk, run_ind+run_pages+nrun_pages-1) == flag_dirty); - arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, - false, true); + arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); /* If the successor is dirty, remove it from runs_dirty. */ if (flag_dirty != 0) { @@ -1202,8 +1037,7 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == prun_size); assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); - arena_avail_remove(arena, chunk, run_ind, prun_pages, true, - false); + arena_avail_remove(arena, chunk, run_ind, prun_pages); /* If the predecessor is dirty, remove it from runs_dirty. */ if (flag_dirty != 0) { @@ -1282,7 +1116,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); assert(arena_mapbits_dirty_get(chunk, run_ind) == arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); - arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); + arena_avail_insert(arena, chunk, run_ind, run_pages); if (dirty) { /* Insert into runs_dirty list. */ @@ -2412,7 +2246,6 @@ arena_new(arena_t *arena, unsigned ind) arena->dss_prec = chunk_dss_prec_get(); /* Initialize chunks. */ - arena_chunk_dirty_new(&arena->chunks_dirty); ql_new(&arena->runs_dirty); arena->spare = NULL; From e8a2fd83a2ddc082fcd4e49373ea05bd79213c71 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Mon, 21 Jul 2014 20:00:14 -0700 Subject: [PATCH 073/352] arena->npurgatory is no longer needed since we drop arena's lock after stashing all the purgeable runs. --- include/jemalloc/internal/arena.h | 8 -------- src/arena.c | 15 +++------------ 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index f87dfe4d..1e2e9876 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -343,14 +343,6 @@ struct arena_s { */ size_t ndirty; - /* - * Approximate number of pages being purged. It is possible for - * multiple threads to purge dirty pages concurrently, and they use - * npurgatory to indicate the total number of pages all threads are - * attempting to purge. - */ - size_t npurgatory; - /* * Size/address-ordered trees of this arena's available runs. The trees * are used for first-best-fit run allocation. diff --git a/src/arena.c b/src/arena.c index 24ed2ba2..68b156bf 100644 --- a/src/arena.c +++ b/src/arena.c @@ -757,10 +757,7 @@ arena_maybe_purge(arena_t *arena) /* Don't purge if the option is disabled. */ if (opt_lg_dirty_mult < 0) return; - /* Don't purge if all dirty pages are already being purged. */ - if (arena->ndirty <= arena->npurgatory) - return; - npurgeable = arena->ndirty - arena->npurgatory; + npurgeable = arena->ndirty; threshold = (arena->nactive >> opt_lg_dirty_mult); /* * Don't purge unless the number of purgeable pages exceeds the @@ -803,7 +800,7 @@ arena_compute_npurgatory(arena_t *arena, bool all) * Compute the minimum number of pages that this thread should try to * purge. */ - npurgeable = arena->ndirty - arena->npurgatory; + npurgeable = arena->ndirty; if (all == false) { size_t threshold = (arena->nactive >> opt_lg_dirty_mult); @@ -942,9 +939,7 @@ arena_purge(arena_t *arena, bool all) size_t ndirty = arena_dirty_count(arena); assert(ndirty == arena->ndirty); } - assert(arena->ndirty > arena->npurgatory || all); - assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - - arena->npurgatory) || all); + assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all); if (config_stats) arena->stats.npurge++; @@ -955,14 +950,11 @@ arena_purge(arena_t *arena, bool all) * reduce ndirty below the threshold. */ npurgatory = arena_compute_npurgatory(arena, all); - arena->npurgatory += npurgatory; ql_new(&purge_list); npurgeable = arena_stash_dirty(arena, all, npurgatory, &purge_list); assert(npurgeable >= npurgatory); - /* Actually we no longer need arena->npurgatory. */ - arena->npurgatory -= npurgatory; npurged = arena_purge_stashed(arena, &purge_list); assert(npurged == npurgeable); @@ -2251,7 +2243,6 @@ arena_new(arena_t *arena, unsigned ind) arena->nactive = 0; arena->ndirty = 0; - arena->npurgatory = 0; arena_avail_tree_new(&arena->runs_avail); From 070b3c3fbd90296610005c111ec6060e8bb23d31 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 14 Aug 2014 14:45:58 -0700 Subject: [PATCH 074/352] Fix and refactor runs_dirty-based purging. Fix runs_dirty-based purging to also purge dirty pages in the spare chunk. Refactor runs_dirty manipulation into arena_dirty_{insert,remove}(), and move the arena->ndirty accounting into those functions. Remove the u.ql_link field from arena_chunk_map_t, and get rid of the enclosing union for u.rb_link, since only rb_link remains. Remove the ndirty field from arena_chunk_t. --- include/jemalloc/internal/arena.h | 34 ++---- src/arena.c | 184 +++++++++++++----------------- 2 files changed, 91 insertions(+), 127 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 1e2e9876..9351e3b0 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -65,23 +65,14 @@ struct arena_chunk_map_s { */ union { #endif - union { - /* - * Linkage for run trees. There are two disjoint uses: - * - * 1) arena_t's runs_avail tree. - * 2) arena_run_t conceptually uses this linkage for in-use - * non-full runs, rather than directly embedding linkage. - */ - rb_node(arena_chunk_map_t) rb_link; - /* - * List of runs currently in purgatory. arena_chunk_purge() - * temporarily allocates runs that contain dirty pages while - * purging, so that other threads cannot use the runs while the - * purging thread is operating without the arena lock held. - */ - ql_elm(arena_chunk_map_t) ql_link; - } u; + /* + * Linkage for run trees. There are two disjoint uses: + * + * 1) arena_t's runs_avail tree. + * 2) arena_run_t conceptually uses this linkage for in-use non-full + * runs, rather than directly embedding linkage. + */ + rb_node(arena_chunk_map_t) rb_link; /* Profile counters, used for large object runs. */ prof_ctx_t *prof_ctx; @@ -167,9 +158,6 @@ struct arena_chunk_s { /* Arena that owns the chunk. */ arena_t *arena; - /* Number of dirty pages. */ - size_t ndirty; - /* * Map of pages within chunk that keeps track of free/large/small. The * first map_bias entries are omitted, since the chunk header does not @@ -317,9 +305,6 @@ struct arena_s { dss_prec_t dss_prec; - /* List of dirty runs this arena manages. */ - arena_chunk_mapelms_t runs_dirty; - /* * In order to avoid rapid chunk allocation/deallocation when an arena * oscillates right on the cusp of needing a new chunk, cache the most @@ -349,6 +334,9 @@ struct arena_s { */ arena_avail_tree_t runs_avail; + /* List of dirty runs this arena manages. */ + arena_chunk_mapelms_t runs_dirty; + /* * user-configureable chunk allocation and deallocation functions. */ diff --git a/src/arena.c b/src/arena.c index 68b156bf..1263269e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -90,7 +90,7 @@ arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) /* Generate red-black tree functions. */ rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, - u.rb_link, arena_run_comp) + rb_link, arena_run_comp) static inline int arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) @@ -123,7 +123,7 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) /* Generate red-black tree functions. */ rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, - u.rb_link, arena_avail_comp) + rb_link, arena_avail_comp) static void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, @@ -132,12 +132,6 @@ arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); - - if (arena_mapbits_dirty_get(chunk, pageind) != 0) { - arena->ndirty += npages; - chunk->ndirty += npages; - } - arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, pageind)); } @@ -149,16 +143,39 @@ arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); - - if (arena_mapbits_dirty_get(chunk, pageind) != 0) { - arena->ndirty -= npages; - chunk->ndirty -= npages; - } - arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, pageind)); } +static void +arena_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages) +{ + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); + assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == + CHUNK_MAP_DIRTY); + ql_elm_new(mapelm, dr_link); + ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); + arena->ndirty += npages; +} + +static void +arena_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages) +{ + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); + assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == + CHUNK_MAP_DIRTY); + ql_remove(&arena->runs_dirty, mapelm, dr_link); + arena->ndirty -= npages; +} + static inline void * arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) { @@ -252,7 +269,6 @@ static void arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, size_t flag_dirty, size_t need_pages) { - arena_chunk_map_t *mapelm; size_t total_pages, rem_pages; total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> @@ -263,11 +279,8 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, rem_pages = total_pages - need_pages; arena_avail_remove(arena, chunk, run_ind, total_pages); - if (flag_dirty != 0) { - /* If the run is dirty, it must be in the dirty list. */ - mapelm = arena_mapp_get(chunk, run_ind); - ql_remove(&arena->runs_dirty, mapelm, dr_link); - } + if (flag_dirty != 0) + arena_dirty_remove(arena, chunk, run_ind, total_pages); arena_cactive_update(arena, need_pages, 0); arena->nactive += need_pages; @@ -280,14 +293,8 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, (rem_pages << LG_PAGE), flag_dirty); - mapelm = arena_mapp_get(chunk, run_ind+need_pages); - /* - * Append the trailing run at the end of the dirty list. - * We could also insert the run at the original place. - * Let us consider this later. - */ - ql_elm_new(mapelm, dr_link); - ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); + arena_dirty_insert(arena, chunk, run_ind+need_pages, + rem_pages); } else { arena_mapbits_unallocated_set(chunk, run_ind+need_pages, (rem_pages << LG_PAGE), @@ -512,11 +519,6 @@ arena_chunk_init_hard(arena_t *arena) chunk->arena = arena; - /* - * Claim that no pages are in use, since the header is merely overhead. - */ - chunk->ndirty = 0; - /* * Initialize the map to contain one maximal free untouched run. Mark * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. @@ -568,11 +570,6 @@ arena_chunk_alloc(arena_t *arena) /* Insert the run into the runs_avail tree. */ arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias); - if (arena_mapbits_dirty_get(chunk, map_bias) != 0) { - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias); - ql_elm_new(mapelm, dr_link); - ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); - } return (chunk); } @@ -626,15 +623,15 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) * it. */ arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias); - if (arena_mapbits_dirty_get(chunk, map_bias) != 0) { - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, map_bias); - ql_remove(&arena->runs_dirty, mapelm, dr_link); - } if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; arena->spare = chunk; + if (arena_mapbits_dirty_get(spare, map_bias) != 0) { + arena_dirty_remove(arena, spare, map_bias, + chunk_npages-map_bias); + } arena_chunk_dalloc_internal(arena, spare); } else arena->spare = chunk; @@ -752,18 +749,17 @@ arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) static inline void arena_maybe_purge(arena_t *arena) { - size_t npurgeable, threshold; + size_t threshold; /* Don't purge if the option is disabled. */ if (opt_lg_dirty_mult < 0) return; - npurgeable = arena->ndirty; threshold = (arena->nactive >> opt_lg_dirty_mult); /* * Don't purge unless the number of purgeable pages exceeds the * threshold. */ - if (npurgeable <= threshold) + if (arena->ndirty <= threshold) return; arena_purge(arena, false); @@ -792,50 +788,53 @@ arena_dirty_count(arena_t *arena) } static size_t -arena_compute_npurgatory(arena_t *arena, bool all) +arena_compute_npurge(arena_t *arena, bool all) { - size_t npurgatory, npurgeable; + size_t npurge; /* * Compute the minimum number of pages that this thread should try to * purge. */ - npurgeable = arena->ndirty; - if (all == false) { size_t threshold = (arena->nactive >> opt_lg_dirty_mult); - npurgatory = npurgeable - threshold; + npurge = arena->ndirty - threshold; } else - npurgatory = npurgeable; + npurge = arena->ndirty; - return (npurgatory); + return (npurge); } static size_t -arena_stash_dirty(arena_t *arena, bool all, size_t npurgatory, +arena_stash_dirty(arena_t *arena, bool all, size_t npurge, arena_chunk_mapelms_t *mapelms) { arena_chunk_map_t *mapelm; size_t nstashed = 0; - arena_chunk_t *chunk; - size_t pageind, npages, run_size; - arena_run_t *run; - /* Add at least npurgatory pages to purge_list. */ + /* Add at least npurge pages to purge_list. */ for (mapelm = ql_first(&arena->runs_dirty); mapelm != NULL; mapelm = ql_first(&arena->runs_dirty)) { - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = arena_mapelm_to_pageind(mapelm); - run_size = arena_mapbits_unallocated_size_get(chunk, pageind); - npages = run_size >> LG_PAGE; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << - LG_PAGE)); + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); + size_t pageind = arena_mapelm_to_pageind(mapelm); + size_t run_size = arena_mapbits_unallocated_size_get(chunk, + pageind); + size_t npages = run_size >> LG_PAGE; + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)(pageind << LG_PAGE)); assert(pageind + npages <= chunk_npages); assert(arena_mapbits_dirty_get(chunk, pageind) == arena_mapbits_dirty_get(chunk, pageind+npages-1)); + /* + * If purging the spare chunk's run, make it available prior to + * allocation. + */ + if (chunk == arena->spare) + arena_chunk_alloc(arena); + /* Temporarily allocate the free dirty run. */ arena_run_split_large(arena, run, run_size, false); /* Append to purge_list for later processing. */ @@ -844,7 +843,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurgatory, nstashed += npages; - if (all == false && nstashed >= npurgatory) + if (all == false && nstashed >= npurge) break; } @@ -856,8 +855,6 @@ arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms) { size_t npurged, nmadvise; arena_chunk_map_t *mapelm; - arena_chunk_t *chunk; - size_t pageind, npages, run_size; if (config_stats) nmadvise = 0; @@ -866,8 +863,9 @@ arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms) malloc_mutex_unlock(&arena->lock); ql_foreach(mapelm, mapelms, dr_link) { + arena_chunk_t *chunk; + size_t pageind, run_size, npages, flag_unzeroed, i; bool unzeroed; - size_t flag_unzeroed, i; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); pageind = arena_mapelm_to_pageind(mapelm); @@ -913,17 +911,14 @@ static void arena_unstash_purged(arena_t *arena, arena_chunk_mapelms_t *mapelms) { arena_chunk_map_t *mapelm; - arena_chunk_t *chunk; - arena_run_t *run; - size_t pageind; /* Deallocate runs. */ for (mapelm = ql_first(mapelms); mapelm != NULL; mapelm = ql_first(mapelms)) { - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = arena_mapelm_to_pageind(mapelm); - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << - LG_PAGE)); + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); + size_t pageind = arena_mapelm_to_pageind(mapelm); + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)(pageind << LG_PAGE)); ql_remove(mapelms, mapelm, dr_link); arena_run_dalloc(arena, run, false, true); } @@ -932,7 +927,7 @@ arena_unstash_purged(arena_t *arena, arena_chunk_mapelms_t *mapelms) void arena_purge(arena_t *arena, bool all) { - size_t npurgatory, npurgeable, npurged; + size_t npurge, npurgeable, npurged; arena_chunk_mapelms_t purge_list; if (config_debug) { @@ -944,21 +939,12 @@ arena_purge(arena_t *arena, bool all) if (config_stats) arena->stats.npurge++; - /* - * Add the minimum number of pages this thread should try to purge to - * arena->npurgatory. This will keep multiple threads from racing to - * reduce ndirty below the threshold. - */ - npurgatory = arena_compute_npurgatory(arena, all); - + npurge = arena_compute_npurge(arena, all); ql_new(&purge_list); - - npurgeable = arena_stash_dirty(arena, all, npurgatory, &purge_list); - assert(npurgeable >= npurgatory); - + npurgeable = arena_stash_dirty(arena, all, npurge, &purge_list); + assert(npurgeable >= npurge); npurged = arena_purge_stashed(arena, &purge_list); assert(npurged == npurgeable); - arena_unstash_purged(arena, &purge_list); } @@ -999,9 +985,8 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, /* If the successor is dirty, remove it from runs_dirty. */ if (flag_dirty != 0) { - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, - run_ind+run_pages); - ql_remove(&arena->runs_dirty, mapelm, dr_link); + arena_dirty_remove(arena, chunk, run_ind+run_pages, + nrun_pages); } size += nrun_size; @@ -1032,11 +1017,8 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, arena_avail_remove(arena, chunk, run_ind, prun_pages); /* If the predecessor is dirty, remove it from runs_dirty. */ - if (flag_dirty != 0) { - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, - run_ind); - ql_remove(&arena->runs_dirty, mapelm, dr_link); - } + if (flag_dirty != 0) + arena_dirty_remove(arena, chunk, run_ind, prun_pages); size += prun_size; run_pages += prun_pages; @@ -1055,7 +1037,6 @@ static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) { arena_chunk_t *chunk; - arena_chunk_map_t *mapelm; size_t size, run_ind, run_pages, flag_dirty; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); @@ -1110,12 +1091,8 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); arena_avail_insert(arena, chunk, run_ind, run_pages); - if (dirty) { - /* Insert into runs_dirty list. */ - mapelm = arena_mapp_get(chunk, run_ind); - ql_elm_new(mapelm, dr_link); - ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); - } + if (dirty) + arena_dirty_insert(arena, chunk, run_ind, run_pages); /* Deallocate chunk if it is now completely unused. */ if (size == arena_maxclass) { @@ -2237,14 +2214,13 @@ arena_new(arena_t *arena, unsigned ind) arena->dss_prec = chunk_dss_prec_get(); - /* Initialize chunks. */ - ql_new(&arena->runs_dirty); arena->spare = NULL; arena->nactive = 0; arena->ndirty = 0; arena_avail_tree_new(&arena->runs_avail); + ql_new(&arena->runs_dirty); /* Initialize bins. */ for (i = 0; i < NBINS; i++) { From 586c8ede42d7d0545d36d9cbb0235fb39221ef3e Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 15 Aug 2014 12:20:20 -0700 Subject: [PATCH 075/352] Fix arena..dss mallctl to handle read-only calls. --- src/ctl.c | 52 +++++++++++++++++++++++++-------------------- test/unit/mallctl.c | 13 ++++++++++++ 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/src/ctl.c b/src/ctl.c index a193605d..fa52a6cc 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1327,45 +1327,51 @@ static int arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - int ret, i; - bool match, err; - const char *dss; + int ret; + const char *dss = NULL; unsigned arena_ind = mib[1]; dss_prec_t dss_prec_old = dss_prec_limit; dss_prec_t dss_prec = dss_prec_limit; malloc_mutex_lock(&ctl_mtx); WRITE(dss, const char *); - match = false; - for (i = 0; i < dss_prec_limit; i++) { - if (strcmp(dss_prec_names[i], dss) == 0) { - dss_prec = i; - match = true; - break; + if (dss != NULL) { + int i; + bool match = false; + + for (i = 0; i < dss_prec_limit; i++) { + if (strcmp(dss_prec_names[i], dss) == 0) { + dss_prec = i; + match = true; + break; + } + } + + if (match == false) { + ret = EINVAL; + goto label_return; } - } - if (match == false) { - ret = EINVAL; - goto label_return; } if (arena_ind < ctl_stats.narenas) { arena_t *arena = arenas[arena_ind]; - if (arena != NULL) { - dss_prec_old = arena_dss_prec_get(arena); - err = arena_dss_prec_set(arena, dss_prec); - } else - err = true; + if (arena == NULL || (dss_prec != dss_prec_limit && + arena_dss_prec_set(arena, dss_prec))) { + ret = EFAULT; + goto label_return; + } + dss_prec_old = arena_dss_prec_get(arena); } else { + if (dss_prec != dss_prec_limit && + chunk_dss_prec_set(dss_prec)) { + ret = EFAULT; + goto label_return; + } dss_prec_old = chunk_dss_prec_get(); - err = chunk_dss_prec_set(dss_prec); } + dss = dss_prec_names[dss_prec_old]; READ(dss, const char *); - if (err) { - ret = EFAULT; - goto label_return; - } ret = 0; label_return: diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 7a8b55f5..c70473cc 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -268,12 +268,25 @@ TEST_BEGIN(test_arena_i_dss) assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_new, &sz, &dss_prec_old, sizeof(dss_prec_old)), 0, "Unexpected mallctl() failure"); + assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_str_ne(dss_prec_old, "primary", + "Unexpected value for dss precedence"); + mib[1] = narenas_total_get(); dss_prec_new = "disabled"; assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, &dss_prec_new, sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure"); assert_str_ne(dss_prec_old, "primary", "Unexpected default for dss precedence"); + + assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_new, &sz, &dss_prec_old, + sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure"); + + assert_d_eq(mallctlbymib(mib, miblen, &dss_prec_old, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_str_ne(dss_prec_old, "primary", + "Unexpected value for dss precedence"); } TEST_END From b41ccdb125b312d4522da1a80091a0137773c964 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 15 Aug 2014 15:01:15 -0700 Subject: [PATCH 076/352] Convert prof_tdata_t's bt2cnt to a comprehensive map. Treat prof_tdata_t's bt2cnt as a comprehensive map of the thread's extant allocation samples (do not limit the total number of entries). This helps prepare the way for per thread heap profiling. --- include/jemalloc/internal/prof.h | 24 ++++-------- src/prof.c | 67 ++++++++------------------------ 2 files changed, 25 insertions(+), 66 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index d82fbc4f..96db4c3e 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -23,9 +23,6 @@ typedef struct prof_tdata_s prof_tdata_t; */ #define PROF_BT_MAX 128 -/* Maximum number of backtraces to store in each per thread LRU cache. */ -#define PROF_TCMAX 1024 - /* Initial hash table size. */ #define PROF_CKH_MINITEMS 64 @@ -87,9 +84,6 @@ struct prof_thr_cnt_s { /* Linkage into prof_ctx_t's cnts_ql. */ ql_elm(prof_thr_cnt_t) cnts_link; - /* Linkage into thread's LRU. */ - ql_elm(prof_thr_cnt_t) lru_link; - /* * Associated context. If a thread frees an object that it did not * allocate, it is possible that the context is not cached in the @@ -157,10 +151,11 @@ typedef ql_head(prof_ctx_t) prof_ctx_list_t; struct prof_tdata_s { /* - * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread keeps a - * cache of backtraces, with associated thread-specific prof_thr_cnt_t - * objects. Other threads may read the prof_thr_cnt_t contents, but no - * others will ever write them. + * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread tracks + * backtraces for which it has non-zero allocation/deallocation counters + * associated with thread-specific prof_thr_cnt_t objects. Other + * threads may read the prof_thr_cnt_t contents, but no others will ever + * write them. * * Upon thread exit, the thread must merge all the prof_thr_cnt_t * counter data into the associated prof_ctx_t objects, and unlink/free @@ -168,12 +163,6 @@ struct prof_tdata_s { */ ckh_t bt2cnt; - /* LRU for contents of bt2cnt. */ - ql_head(prof_thr_cnt_t) lru_ql; - - /* Backtrace vector, used for calls to prof_backtrace(). */ - void **vec; - /* Sampling state. */ uint64_t prng_state; uint64_t bytes_until_sample; @@ -182,6 +171,9 @@ struct prof_tdata_s { bool enq; bool enq_idump; bool enq_gdump; + + /* Backtrace vector, used for calls to prof_backtrace(). */ + void *vec[PROF_BT_MAX]; }; #endif /* JEMALLOC_H_STRUCTS */ diff --git a/src/prof.c b/src/prof.c index 0eb7dbdb..4f95fdb9 100644 --- a/src/prof.c +++ b/src/prof.c @@ -567,33 +567,13 @@ prof_lookup(prof_bt_t *bt) return (NULL); /* Link a prof_thd_cnt_t into ctx for this thread. */ - if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) { - assert(ckh_count(&prof_tdata->bt2cnt) > 0); - /* - * Flush the least recently used cnt in order to keep - * bt2cnt from becoming too large. - */ - ret.p = ql_last(&prof_tdata->lru_ql, lru_link); - assert(ret.v != NULL); - if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt, - NULL, NULL)) - not_reached(); - ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); - prof_ctx_merge(ret.p->ctx, ret.p); - /* ret can now be re-used. */ - } else { - assert(ckh_count(&prof_tdata->bt2cnt) < PROF_TCMAX); - /* Allocate and partially initialize a new cnt. */ - ret.v = imalloc(sizeof(prof_thr_cnt_t)); - if (ret.p == NULL) { - if (new_ctx) - prof_ctx_destroy(ctx); - return (NULL); - } - ql_elm_new(ret.p, cnts_link); - ql_elm_new(ret.p, lru_link); + ret.v = imalloc(sizeof(prof_thr_cnt_t)); + if (ret.p == NULL) { + if (new_ctx) + prof_ctx_destroy(ctx); + return (NULL); } - /* Finish initializing ret. */ + ql_elm_new(ret.p, cnts_link); ret.p->ctx = ctx; ret.p->epoch = 0; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); @@ -603,15 +583,10 @@ prof_lookup(prof_bt_t *bt) idalloc(ret.v); return (NULL); } - ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); malloc_mutex_lock(ctx->lock); ql_tail_insert(&ctx->cnts_ql, ret.p, cnts_link); ctx->nlimbo--; malloc_mutex_unlock(ctx->lock); - } else { - /* Move ret to the front of the LRU. */ - ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); - ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); } return (ret.p); @@ -1247,14 +1222,6 @@ prof_tdata_init(void) idalloc(prof_tdata); return (NULL); } - ql_new(&prof_tdata->lru_ql); - - prof_tdata->vec = imalloc(sizeof(void *) * PROF_BT_MAX); - if (prof_tdata->vec == NULL) { - ckh_delete(&prof_tdata->bt2cnt); - idalloc(prof_tdata); - return (NULL); - } prof_tdata->prng_state = (uint64_t)(uintptr_t)prof_tdata; prof_sample_threshold_update(prof_tdata); @@ -1271,7 +1238,6 @@ prof_tdata_init(void) void prof_tdata_cleanup(void *arg) { - prof_thr_cnt_t *cnt; prof_tdata_t *prof_tdata = *(prof_tdata_t **)arg; cassert(config_prof); @@ -1292,21 +1258,22 @@ prof_tdata_cleanup(void *arg) * nothing, so that the destructor will not be called again. */ } else if (prof_tdata != NULL) { - /* - * Delete the hash table. All of its contents can still be - * iterated over via the LRU. - */ - ckh_delete(&prof_tdata->bt2cnt); + union { + prof_thr_cnt_t *p; + void *v; + } cnt; + size_t tabind; + /* * Iteratively merge cnt's into the global stats and delete * them. */ - while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { - ql_remove(&prof_tdata->lru_ql, cnt, lru_link); - prof_ctx_merge(cnt->ctx, cnt); - idalloc(cnt); + for (tabind = 0; ckh_iter(&prof_tdata->bt2cnt, &tabind, NULL, + &cnt.v);) { + prof_ctx_merge(cnt.p->ctx, cnt.p); + idalloc(cnt.v); } - idalloc(prof_tdata->vec); + ckh_delete(&prof_tdata->bt2cnt); idalloc(prof_tdata); prof_tdata = PROF_TDATA_STATE_PURGATORY; prof_tdata_tsd_set(&prof_tdata); From ab532e97991d190e9368781cf308c60c2319b933 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 15 Aug 2014 15:05:12 -0700 Subject: [PATCH 077/352] Directly embed prof_ctx_t's bt. --- include/jemalloc/internal/prof.h | 13 +++--- src/prof.c | 69 +++++++++----------------------- 2 files changed, 26 insertions(+), 56 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 96db4c3e..9be908d6 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -115,9 +115,6 @@ struct prof_thr_cnt_s { }; struct prof_ctx_s { - /* Associated backtrace. */ - prof_bt_t *bt; - /* Protects nlimbo, cnt_merged, and cnts_ql. */ malloc_mutex_t *lock; @@ -146,6 +143,12 @@ struct prof_ctx_s { /* Linkage for list of contexts to be dumped. */ ql_elm(prof_ctx_t) dump_link; + + /* Associated backtrace. */ + prof_bt_t bt; + + /* Backtrace vector, variable size, referred to by bt. */ + void *vec[1]; }; typedef ql_head(prof_ctx_t) prof_ctx_list_t; @@ -425,7 +428,7 @@ prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, } if ((uintptr_t)old_ctx > (uintptr_t)1U) { - told_cnt = prof_lookup(old_ctx->bt); + told_cnt = prof_lookup(&old_ctx->bt); if (told_cnt == NULL) { /* * It's too late to propagate OOM for this realloc(), @@ -483,7 +486,7 @@ prof_free(const void *ptr, size_t size) if ((uintptr_t)ctx > (uintptr_t)1) { prof_thr_cnt_t *tcnt; assert(size == isalloc(ptr, true)); - tcnt = prof_lookup(ctx->bt); + tcnt = prof_lookup(&ctx->bt); if (tcnt != NULL) { tcnt->epoch++; diff --git a/src/prof.c b/src/prof.c index 4f95fdb9..1b396afe 100644 --- a/src/prof.c +++ b/src/prof.c @@ -87,41 +87,6 @@ bt_init(prof_bt_t *bt, void **vec) bt->len = 0; } -static void -bt_destroy(prof_bt_t *bt) -{ - - cassert(config_prof); - - idalloc(bt); -} - -static prof_bt_t * -bt_dup(prof_bt_t *bt) -{ - prof_bt_t *ret; - - cassert(config_prof); - - /* - * Create a single allocation that has space for vec immediately - * following the prof_bt_t structure. The backtraces that get - * stored in the backtrace caches are copied from stack-allocated - * temporary variables, so size is known at creation time. Making this - * a contiguous object improves cache locality. - */ - ret = (prof_bt_t *)imalloc(QUANTUM_CEILING(sizeof(prof_bt_t)) + - (bt->len * sizeof(void *))); - if (ret == NULL) - return (NULL); - ret->vec = (void **)((uintptr_t)ret + - QUANTUM_CEILING(sizeof(prof_bt_t))); - memcpy(ret->vec, bt->vec, bt->len * sizeof(void *)); - ret->len = bt->len; - - return (ret); -} - static inline void prof_enter(prof_tdata_t *prof_tdata) { @@ -388,11 +353,16 @@ prof_ctx_mutex_choose(void) return (&ctx_locks[(nctxs - 1) % PROF_NCTX_LOCKS]); } -static void -prof_ctx_init(prof_ctx_t *ctx, prof_bt_t *bt) +static prof_ctx_t * +prof_ctx_create(prof_bt_t *bt) { - - ctx->bt = bt; + /* + * Create a single allocation that has space for vec of length bt->len. + */ + prof_ctx_t *ctx = (prof_ctx_t *)imalloc(offsetof(prof_ctx_t, vec) + + (bt->len * sizeof(void *))); + if (ctx == NULL) + return (NULL); ctx->lock = prof_ctx_mutex_choose(); /* * Set nlimbo to 1, in order to avoid a race condition with @@ -402,6 +372,11 @@ prof_ctx_init(prof_ctx_t *ctx, prof_bt_t *bt) ql_elm_new(ctx, dump_link); memset(&ctx->cnt_merged, 0, sizeof(prof_cnt_t)); ql_new(&ctx->cnts_ql); + /* Duplicate bt. */ + memcpy(ctx->vec, bt->vec, bt->len * sizeof(void *)); + ctx->bt.vec = ctx->vec; + ctx->bt.len = bt->len; + return (ctx); } static void @@ -428,12 +403,11 @@ prof_ctx_destroy(prof_ctx_t *ctx) assert(ctx->cnt_merged.accumobjs == 0); assert(ctx->cnt_merged.accumbytes == 0); /* Remove ctx from bt2ctx. */ - if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) + if (ckh_remove(&bt2ctx, &ctx->bt, NULL, NULL)) not_reached(); prof_leave(prof_tdata); /* Destroy ctx. */ malloc_mutex_unlock(ctx->lock); - bt_destroy(ctx->bt); idalloc(ctx); } else { /* @@ -501,22 +475,15 @@ prof_lookup_global(prof_bt_t *bt, prof_tdata_t *prof_tdata, void **p_btkey, prof_enter(prof_tdata); if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { /* bt has never been seen before. Insert it. */ - ctx.v = imalloc(sizeof(prof_ctx_t)); + ctx.p = prof_ctx_create(bt); if (ctx.v == NULL) { prof_leave(prof_tdata); return (true); } - btkey.p = bt_dup(bt); - if (btkey.v == NULL) { - prof_leave(prof_tdata); - idalloc(ctx.v); - return (true); - } - prof_ctx_init(ctx.p, btkey.p); + btkey.p = &ctx.p->bt; if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { /* OOM. */ prof_leave(prof_tdata); - idalloc(btkey.v); idalloc(ctx.v); return (true); } @@ -1039,7 +1006,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) /* Dump per ctx profile stats. */ while ((ctx.p = ql_first(&ctx_ql)) != NULL) { - if (prof_dump_ctx(propagate_err, ctx.p, ctx.p->bt, &ctx_ql)) + if (prof_dump_ctx(propagate_err, ctx.p, &ctx.p->bt, &ctx_ql)) goto label_write_error; } From 3a81cbd2d4f2d8c052f11f4b0b73ee5c84a33d4f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 16 Aug 2014 12:58:55 -0700 Subject: [PATCH 078/352] Dump heap profile backtraces in a stable order. Also iterate over per thread stats in a stable order, which prepares the way for stable ordering of per thread heap profile dumps. --- include/jemalloc/internal/prof.h | 24 +++-- src/prof.c | 157 +++++++++++++++++++++---------- 2 files changed, 119 insertions(+), 62 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 9be908d6..9398ad91 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -1,6 +1,7 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES +typedef uint64_t prof_thr_uid_t; typedef struct prof_bt_s prof_bt_t; typedef struct prof_cnt_s prof_cnt_t; typedef struct prof_thr_cnt_s prof_thr_cnt_t; @@ -81,15 +82,17 @@ struct prof_cnt_s { }; struct prof_thr_cnt_s { - /* Linkage into prof_ctx_t's cnts_ql. */ - ql_elm(prof_thr_cnt_t) cnts_link; + prof_thr_uid_t thr_uid; + + /* Linkage into prof_ctx_t's thr_cnts. */ + rb_node(prof_thr_cnt_t) thr_cnt_link; /* * Associated context. If a thread frees an object that it did not - * allocate, it is possible that the context is not cached in the + * allocate, it is possible that the context is not present in the * thread's hash table, in which case it must be able to look up the * context, insert a new prof_thr_cnt_t into the thread's hash table, - * and link it into the prof_ctx_t's cnts_ql. + * and link it into the prof_ctx_t's thr_cnts. */ prof_ctx_t *ctx; @@ -113,9 +116,10 @@ struct prof_thr_cnt_s { /* Profiling counters. */ prof_cnt_t cnts; }; +typedef rb_tree(prof_thr_cnt_t) prof_thr_cnt_tree_t; struct prof_ctx_s { - /* Protects nlimbo, cnt_merged, and cnts_ql. */ + /* Protects nlimbo, cnt_merged, and thr_cnts. */ malloc_mutex_t *lock; /* @@ -136,13 +140,13 @@ struct prof_ctx_s { prof_cnt_t cnt_merged; /* - * List of profile counters, one for each thread that has allocated in + * Tree of profile counters, one for each thread that has allocated in * this context. */ - ql_head(prof_thr_cnt_t) cnts_ql; + prof_thr_cnt_tree_t thr_cnts; - /* Linkage for list of contexts to be dumped. */ - ql_elm(prof_ctx_t) dump_link; + /* Linkage for tree of contexts to be dumped. */ + rb_node(prof_ctx_t) dump_link; /* Associated backtrace. */ prof_bt_t bt; @@ -150,7 +154,7 @@ struct prof_ctx_s { /* Backtrace vector, variable size, referred to by bt. */ void *vec[1]; }; -typedef ql_head(prof_ctx_t) prof_ctx_list_t; +typedef rb_tree(prof_ctx_t) prof_ctx_tree_t; struct prof_tdata_s { /* diff --git a/src/prof.c b/src/prof.c index 1b396afe..497ccf42 100644 --- a/src/prof.c +++ b/src/prof.c @@ -77,6 +77,33 @@ static bool prof_booted = false; /******************************************************************************/ +JEMALLOC_INLINE_C int +prof_thr_cnt_comp(const prof_thr_cnt_t *a, const prof_thr_cnt_t *b) +{ + prof_thr_uid_t a_uid = a->thr_uid; + prof_thr_uid_t b_uid = b->thr_uid; + + return ((a_uid > b_uid) - (a_uid < b_uid)); +} + +rb_gen(static UNUSED, thr_cnt_tree_, prof_thr_cnt_tree_t, prof_thr_cnt_t, + thr_cnt_link, prof_thr_cnt_comp) + +JEMALLOC_INLINE_C int +prof_ctx_comp(const prof_ctx_t *a, const prof_ctx_t *b) +{ + unsigned a_len = a->bt.len; + unsigned b_len = b->bt.len; + unsigned comp_len = (a_len < b_len) ? a_len : b_len; + int ret = memcmp(a->bt.vec, b->bt.vec, comp_len * sizeof(void *)); + if (ret == 0) + ret = (a_len > b_len) - (a_len < b_len); + return (ret); +} + +rb_gen(static UNUSED, ctx_tree_, prof_ctx_tree_t, prof_ctx_t, dump_link, + prof_ctx_comp) + void bt_init(prof_bt_t *bt, void **vec) { @@ -369,9 +396,8 @@ prof_ctx_create(prof_bt_t *bt) * prof_ctx_merge()/prof_ctx_destroy(). */ ctx->nlimbo = 1; - ql_elm_new(ctx, dump_link); memset(&ctx->cnt_merged, 0, sizeof(prof_cnt_t)); - ql_new(&ctx->cnts_ql); + thr_cnt_tree_new(&ctx->thr_cnts); /* Duplicate bt. */ memcpy(ctx->vec, bt->vec, bt->len * sizeof(void *)); ctx->bt.vec = ctx->vec; @@ -397,8 +423,8 @@ prof_ctx_destroy(prof_ctx_t *ctx) assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX); prof_enter(prof_tdata); malloc_mutex_lock(ctx->lock); - if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 0 && - ctx->nlimbo == 1) { + if (thr_cnt_tree_first(&ctx->thr_cnts) == NULL && + ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 1) { assert(ctx->cnt_merged.curbytes == 0); assert(ctx->cnt_merged.accumobjs == 0); assert(ctx->cnt_merged.accumbytes == 0); @@ -433,9 +459,9 @@ prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) ctx->cnt_merged.curbytes += cnt->cnts.curbytes; ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; - ql_remove(&ctx->cnts_ql, cnt, cnts_link); - if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && - ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) { + thr_cnt_tree_remove(&ctx->thr_cnts, cnt); + if (opt_prof_accum == false && thr_cnt_tree_first(&ctx->thr_cnts) == + NULL && ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) { /* * Increment ctx->nlimbo in order to keep another thread from * winning the race to destroy ctx while this one has ctx->lock @@ -540,7 +566,6 @@ prof_lookup(prof_bt_t *bt) prof_ctx_destroy(ctx); return (NULL); } - ql_elm_new(ret.p, cnts_link); ret.p->ctx = ctx; ret.p->epoch = 0; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); @@ -551,7 +576,7 @@ prof_lookup(prof_bt_t *bt) return (NULL); } malloc_mutex_lock(ctx->lock); - ql_tail_insert(&ctx->cnts_ql, ret.p, cnts_link); + thr_cnt_tree_insert(&ctx->thr_cnts, ret.p); ctx->nlimbo--; malloc_mutex_unlock(ctx->lock); } @@ -745,12 +770,41 @@ prof_dump_printf(bool propagate_err, const char *format, ...) return (ret); } +static prof_thr_cnt_t * +ctx_sum_iter(prof_thr_cnt_tree_t *thr_cnts, prof_thr_cnt_t *thr_cnt, void *arg) +{ + prof_ctx_t *ctx = (prof_ctx_t *)arg; + volatile unsigned *epoch = &thr_cnt->epoch; + prof_cnt_t tcnt; + + while (true) { + unsigned epoch0 = *epoch; + + /* Make sure epoch is even. */ + if (epoch0 & 1U) + continue; + + memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t)); + + /* Terminate if epoch didn't change while reading. */ + if (*epoch == epoch0) + break; + } + + ctx->cnt_summed.curobjs += tcnt.curobjs; + ctx->cnt_summed.curbytes += tcnt.curbytes; + if (opt_prof_accum) { + ctx->cnt_summed.accumobjs += tcnt.accumobjs; + ctx->cnt_summed.accumbytes += tcnt.accumbytes; + } + + return (NULL); +} + static void prof_dump_ctx_prep(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx, - prof_ctx_list_t *ctx_ql) + prof_ctx_tree_t *ctxs) { - prof_thr_cnt_t *thr_cnt; - prof_cnt_t tcnt; cassert(config_prof); @@ -762,33 +816,10 @@ prof_dump_ctx_prep(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx, * prof_dump()'s second pass. */ ctx->nlimbo++; - ql_tail_insert(ctx_ql, ctx, dump_link); + ctx_tree_insert(ctxs, ctx); memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); - ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { - volatile unsigned *epoch = &thr_cnt->epoch; - - while (true) { - unsigned epoch0 = *epoch; - - /* Make sure epoch is even. */ - if (epoch0 & 1U) - continue; - - memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t)); - - /* Terminate if epoch didn't change while reading. */ - if (*epoch == epoch0) - break; - } - - ctx->cnt_summed.curobjs += tcnt.curobjs; - ctx->cnt_summed.curbytes += tcnt.curbytes; - if (opt_prof_accum) { - ctx->cnt_summed.accumobjs += tcnt.accumobjs; - ctx->cnt_summed.accumbytes += tcnt.accumbytes; - } - } + thr_cnt_tree_iter(&ctx->thr_cnts, NULL, ctx_sum_iter, (void *)ctx); if (ctx->cnt_summed.curobjs != 0) (*leak_nctx)++; @@ -829,25 +860,24 @@ prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) } static void -prof_dump_ctx_cleanup_locked(prof_ctx_t *ctx, prof_ctx_list_t *ctx_ql) +prof_dump_ctx_cleanup_locked(prof_ctx_t *ctx, prof_ctx_tree_t *ctxs) { ctx->nlimbo--; - ql_remove(ctx_ql, ctx, dump_link); } static void -prof_dump_ctx_cleanup(prof_ctx_t *ctx, prof_ctx_list_t *ctx_ql) +prof_dump_ctx_cleanup(prof_ctx_t *ctx, prof_ctx_tree_t *ctxs) { malloc_mutex_lock(ctx->lock); - prof_dump_ctx_cleanup_locked(ctx, ctx_ql); + prof_dump_ctx_cleanup_locked(ctx, ctxs); malloc_mutex_unlock(ctx->lock); } static bool prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, const prof_bt_t *bt, - prof_ctx_list_t *ctx_ql) + prof_ctx_tree_t *ctxs) { bool ret; unsigned i; @@ -895,7 +925,7 @@ prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, const prof_bt_t *bt, ret = false; label_return: - prof_dump_ctx_cleanup_locked(ctx, ctx_ql); + prof_dump_ctx_cleanup_locked(ctx, ctxs); malloc_mutex_unlock(ctx->lock); return (ret); } @@ -966,6 +996,26 @@ prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_nctx, } } +static prof_ctx_t * +prof_ctx_dump_iter(prof_ctx_tree_t *ctxs, prof_ctx_t *ctx, void *arg) +{ + bool propagate_err = *(bool *)arg; + + if (prof_dump_ctx(propagate_err, ctx, &ctx->bt, ctxs)) + return (ctx_tree_next(ctxs, ctx)); + + return (NULL); +} + +static prof_ctx_t * +prof_ctx_cleanup_iter(prof_ctx_tree_t *ctxs, prof_ctx_t *ctx, void *arg) +{ + + prof_dump_ctx_cleanup(ctx, ctxs); + + return (NULL); +} + static bool prof_dump(bool propagate_err, const char *filename, bool leakcheck) { @@ -977,7 +1027,8 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) void *v; } ctx; size_t leak_nctx; - prof_ctx_list_t ctx_ql; + prof_ctx_tree_t ctxs; + prof_ctx_t *cleanup_start = NULL; cassert(config_prof); @@ -990,10 +1041,10 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) /* Merge per thread profile stats, and sum them in cnt_all. */ memset(&cnt_all, 0, sizeof(prof_cnt_t)); leak_nctx = 0; - ql_new(&ctx_ql); + ctx_tree_new(&ctxs); prof_enter(prof_tdata); for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) - prof_dump_ctx_prep(ctx.p, &cnt_all, &leak_nctx, &ctx_ql); + prof_dump_ctx_prep(ctx.p, &cnt_all, &leak_nctx, &ctxs); prof_leave(prof_tdata); /* Create dump file. */ @@ -1005,10 +1056,10 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) goto label_write_error; /* Dump per ctx profile stats. */ - while ((ctx.p = ql_first(&ctx_ql)) != NULL) { - if (prof_dump_ctx(propagate_err, ctx.p, &ctx.p->bt, &ctx_ql)) - goto label_write_error; - } + cleanup_start = ctx_tree_iter(&ctxs, NULL, prof_ctx_dump_iter, + (void *)&propagate_err); + if (cleanup_start != NULL) + goto label_write_error; /* Dump /proc//maps if possible. */ if (prof_dump_maps(propagate_err)) @@ -1026,8 +1077,10 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) label_write_error: prof_dump_close(propagate_err); label_open_close_error: - while ((ctx.p = ql_first(&ctx_ql)) != NULL) - prof_dump_ctx_cleanup(ctx.p, &ctx_ql); + if (cleanup_start != NULL) { + ctx_tree_iter(&ctxs, cleanup_start, prof_ctx_cleanup_iter, + NULL); + } malloc_mutex_unlock(&prof_dump_mtx); return (true); } From 1628e8615ed6c82ded14d6013ac775274eb426e6 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 19 Aug 2014 01:28:49 -0700 Subject: [PATCH 079/352] Add rb_empty(). --- include/jemalloc/internal/rb.h | 13 +++++++++++++ test/unit/rb.c | 3 +++ 2 files changed, 16 insertions(+) diff --git a/include/jemalloc/internal/rb.h b/include/jemalloc/internal/rb.h index 423802eb..ffe3bb0d 100644 --- a/include/jemalloc/internal/rb.h +++ b/include/jemalloc/internal/rb.h @@ -158,6 +158,8 @@ struct { \ #define rb_proto(a_attr, a_prefix, a_rbt_type, a_type) \ a_attr void \ a_prefix##new(a_rbt_type *rbtree); \ +a_attr bool \ +a_prefix##empty(a_rbt_type *rbtree); \ a_attr a_type * \ a_prefix##first(a_rbt_type *rbtree); \ a_attr a_type * \ @@ -224,6 +226,13 @@ a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ * Args: * tree: Pointer to an uninitialized red-black tree object. * + * static bool + * ex_empty(ex_t *tree); + * Description: Determine whether tree is empty. + * Args: + * tree: Pointer to an initialized red-black tree object. + * Ret: True if tree is empty, false otherwise. + * * static ex_node_t * * ex_first(ex_t *tree); * static ex_node_t * @@ -309,6 +318,10 @@ a_attr void \ a_prefix##new(a_rbt_type *rbtree) { \ rb_new(a_type, a_field, rbtree); \ } \ +a_attr bool \ +a_prefix##empty(a_rbt_type *rbtree) { \ + return (rbtree->rbt_root == &rbtree->rbt_nil); \ +} \ a_attr a_type * \ a_prefix##first(a_rbt_type *rbtree) { \ a_type *ret; \ diff --git a/test/unit/rb.c b/test/unit/rb.c index b737485a..e43907f1 100644 --- a/test/unit/rb.c +++ b/test/unit/rb.c @@ -49,6 +49,7 @@ TEST_BEGIN(test_rb_empty) tree_new(&tree); + assert_true(tree_empty(&tree), "Tree should be empty"); assert_ptr_null(tree_first(&tree), "Unexpected node"); assert_ptr_null(tree_last(&tree), "Unexpected node"); @@ -265,6 +266,8 @@ TEST_BEGIN(test_rb_random) assert_u_eq(tree_iterate_reverse(&tree), k+1, "Unexpected node iteration count"); + assert_false(tree_empty(&tree), + "Tree should not be empty"); assert_ptr_not_null(tree_first(&tree), "Tree should not be empty"); assert_ptr_not_null(tree_last(&tree), From 602c8e0971160e4b85b08b16cf8a2375aa24bc04 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 18 Aug 2014 16:22:13 -0700 Subject: [PATCH 080/352] Implement per thread heap profiling. Rename data structures (prof_thr_cnt_t-->prof_tctx_t, prof_ctx_t-->prof_gctx_t), and convert to storing a prof_tctx_t for sampled objects. Convert PROF_ALLOC_PREP() to prof_alloc_prep(), since precise backtrace depth within jemalloc functions is no longer an issue (pprof prunes irrelevant frames). Implement mallctl's: - prof.reset implements full sample data reset, and optional change of sample interval. - prof.lg_sample reads the current sample interval (opt.lg_prof_sample was the permanent source of truth prior to prof.reset). - thread.prof.name provides naming capability for threads within heap profile dumps. - thread.prof.active makes it possible to activate/deactivate heap profiling for individual threads. Modify the heap dump files to contain per thread heap profile data. This change is incompatible with the existing pprof, which will require enhancements to read and process the enriched data. --- doc/jemalloc.xml.in | 56 +- include/jemalloc/internal/arena.h | 22 +- include/jemalloc/internal/extent.h | 2 +- include/jemalloc/internal/huge.h | 4 +- include/jemalloc/internal/private_symbols.txt | 21 +- include/jemalloc/internal/prof.h | 444 +++---- src/ctl.c | 97 +- src/huge.c | 12 +- src/jemalloc.c | 140 +- src/prof.c | 1159 +++++++++++------ src/stats.c | 2 +- 11 files changed, 1235 insertions(+), 724 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 308d0c65..8f4327f3 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1047,7 +1047,7 @@ malloc_conf = "xmalloc:true";]]> opt.lg_prof_sample - (ssize_t) + (size_t) r- [] @@ -1243,6 +1243,35 @@ malloc_conf = "xmalloc:true";]]> the developer may find manual flushing useful. + + + thread.prof.name + (const char *) + rw + [] + + Get/set the descriptive name associated with the calling + thread in memory profile dumps. An internal copy of the name string is + created, so the input string need not be maintained after this interface + completes execution. The output string of this interface should be + copied for non-ephemeral uses, because multiple implementation details + can cause asynchronous string deallocation. + + + + + thread.prof.active + (bool) + rw + [] + + Control whether sampling is currently active for the + calling thread. This is a deactivation mechanism in addition to prof.active; both must + be active for the calling thread to sample. This flag is enabled by + default. + + arena.<i>.purge @@ -1492,6 +1521,31 @@ malloc_conf = "xmalloc:true";]]> option. + + + prof.reset + (size_t) + -w + [] + + Reset all memory profile statistics, and optionally + update the sample rate (see opt.lg_prof_sample). + + + + + + prof.lg_sample + (size_t) + r- + [] + + Get the sample rate (see opt.lg_prof_sample). + + + prof.interval diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 9351e3b0..f3f6426c 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -58,7 +58,7 @@ typedef struct arena_s arena_t; struct arena_chunk_map_s { #ifndef JEMALLOC_PROF /* - * Overlay prof_ctx in order to allow it to be referenced by dead code. + * Overlay prof_tctx in order to allow it to be referenced by dead code. * Such antics aren't warranted for per arena data structures, but * chunk map overhead accounts for a percentage of memory, rather than * being just a fixed cost. @@ -75,7 +75,7 @@ struct arena_chunk_map_s { rb_node(arena_chunk_map_t) rb_link; /* Profile counters, used for large object runs. */ - prof_ctx_t *prof_ctx; + prof_tctx_t *prof_tctx; #ifndef JEMALLOC_PROF }; /* union { ... }; */ #endif @@ -472,8 +472,8 @@ size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); -prof_ctx_t *arena_prof_ctx_get(const void *ptr); -void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +prof_tctx_t *arena_prof_tctx_get(const void *ptr); +void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); @@ -987,10 +987,10 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) return (regind); } -JEMALLOC_INLINE prof_ctx_t * -arena_prof_ctx_get(const void *ptr) +JEMALLOC_INLINE prof_tctx_t * +arena_prof_tctx_get(const void *ptr) { - prof_ctx_t *ret; + prof_tctx_t *ret; arena_chunk_t *chunk; size_t pageind, mapbits; @@ -1003,15 +1003,15 @@ arena_prof_ctx_get(const void *ptr) mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); if ((mapbits & CHUNK_MAP_LARGE) == 0) - ret = (prof_ctx_t *)(uintptr_t)1U; + ret = (prof_tctx_t *)(uintptr_t)1U; else - ret = arena_mapp_get(chunk, pageind)->prof_ctx; + ret = arena_mapp_get(chunk, pageind)->prof_tctx; return (ret); } JEMALLOC_INLINE void -arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { arena_chunk_t *chunk; size_t pageind; @@ -1025,7 +1025,7 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) assert(arena_mapbits_allocated_get(chunk, pageind) != 0); if (arena_mapbits_large_get(chunk, pageind) != 0) - arena_mapp_get(chunk, pageind)->prof_ctx = ctx; + arena_mapp_get(chunk, pageind)->prof_tctx = tctx; } JEMALLOC_ALWAYS_INLINE void * diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 000ef6d5..5b00076f 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -16,7 +16,7 @@ struct extent_node_s { rb_node(extent_node_t) link_ad; /* Profile counters, used for huge objects. */ - prof_ctx_t *prof_ctx; + prof_tctx_t *prof_tctx; /* Pointer to the extent that this tree node is responsible for. */ void *addr; diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 1e545367..2ec77520 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -21,8 +21,8 @@ extern huge_dalloc_junk_t *huge_dalloc_junk; #endif void huge_dalloc(void *ptr); size_t huge_salloc(const void *ptr); -prof_ctx_t *huge_prof_ctx_get(const void *ptr); -void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +prof_tctx_t *huge_prof_tctx_get(const void *ptr); +void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); bool huge_boot(void); void huge_prefork(void); void huge_postfork_parent(void); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 3401301c..13505455 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -48,9 +48,9 @@ arena_prefork arena_prof_accum arena_prof_accum_impl arena_prof_accum_locked -arena_prof_ctx_get -arena_prof_ctx_set arena_prof_promoted +arena_prof_tctx_get +arena_prof_tctx_set arena_ptr_small_binind_get arena_purge_all arena_quarantine_junk_small @@ -208,8 +208,8 @@ huge_palloc huge_postfork_child huge_postfork_parent huge_prefork -huge_prof_ctx_get -huge_prof_ctx_set +huge_prof_tctx_get +huge_prof_tctx_set huge_ralloc huge_ralloc_no_move huge_salloc @@ -287,28 +287,31 @@ opt_zero p2rz pages_purge pow2_ceil +prof_alloc_prep prof_backtrace prof_boot0 prof_boot1 prof_boot2 prof_bt_count -prof_ctx_get -prof_ctx_set prof_dump_open prof_free +prof_free_sampled_object prof_gdump prof_idump prof_interval prof_lookup prof_malloc -prof_malloc_record_object +prof_malloc_sample_object prof_mdump prof_postfork_child prof_postfork_parent prof_prefork prof_realloc +prof_reset prof_sample_accum_update prof_sample_threshold_update +prof_tctx_get +prof_tctx_set prof_tdata_booted prof_tdata_cleanup prof_tdata_get @@ -322,6 +325,10 @@ prof_tdata_tsd_get prof_tdata_tsd_get_wrapper prof_tdata_tsd_init_head prof_tdata_tsd_set +prof_thread_active_get +prof_thread_active_set +prof_thread_name_get +prof_thread_name_set quarantine quarantine_alloc_hook quarantine_boot diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 9398ad91..104bfade 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -1,11 +1,10 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -typedef uint64_t prof_thr_uid_t; typedef struct prof_bt_s prof_bt_t; typedef struct prof_cnt_s prof_cnt_t; -typedef struct prof_thr_cnt_s prof_thr_cnt_t; -typedef struct prof_ctx_s prof_ctx_t; +typedef struct prof_tctx_s prof_tctx_t; +typedef struct prof_gctx_s prof_gctx_t; typedef struct prof_tdata_s prof_tdata_t; /* Option defaults. */ @@ -34,11 +33,17 @@ typedef struct prof_tdata_s prof_tdata_t; #define PROF_PRINTF_BUFSIZE 128 /* - * Number of mutexes shared among all ctx's. No space is allocated for these + * Number of mutexes shared among all gctx's. No space is allocated for these * unless profiling is enabled, so it's okay to over-provision. */ #define PROF_NCTX_LOCKS 1024 +/* + * Number of mutexes shared among all tdata's. No space is allocated for these + * unless profiling is enabled, so it's okay to over-provision. + */ +#define PROF_NTDATA_LOCKS 256 + /* * prof_tdata pointers close to NULL are used to encode state information that * is used for cleaning up during thread shutdown. @@ -66,87 +71,70 @@ typedef struct { #endif struct prof_cnt_s { - /* - * Profiling counters. An allocation/deallocation pair can operate on - * different prof_thr_cnt_t objects that are linked into the same - * prof_ctx_t cnts_ql, so it is possible for the cur* counters to go - * negative. In principle it is possible for the *bytes counters to - * overflow/underflow, but a general solution would require something - * like 128-bit counters; this implementation doesn't bother to solve - * that problem. - */ - int64_t curobjs; - int64_t curbytes; + /* Profiling counters. */ + uint64_t curobjs; + uint64_t curbytes; uint64_t accumobjs; uint64_t accumbytes; }; -struct prof_thr_cnt_s { - prof_thr_uid_t thr_uid; +typedef enum { + prof_tctx_state_nominal, + prof_tctx_state_dumping, + prof_tctx_state_purgatory /* Dumper must finish destroying. */ +} prof_tctx_state_t; - /* Linkage into prof_ctx_t's thr_cnts. */ - rb_node(prof_thr_cnt_t) thr_cnt_link; +struct prof_tctx_s { + /* Thread data for thread that performed the allocation. */ + prof_tdata_t *tdata; - /* - * Associated context. If a thread frees an object that it did not - * allocate, it is possible that the context is not present in the - * thread's hash table, in which case it must be able to look up the - * context, insert a new prof_thr_cnt_t into the thread's hash table, - * and link it into the prof_ctx_t's thr_cnts. - */ - prof_ctx_t *ctx; - - /* - * Threads use memory barriers to update the counters. Since there is - * only ever one writer, the only challenge is for the reader to get a - * consistent read of the counters. - * - * The writer uses this series of operations: - * - * 1) Increment epoch to an odd number. - * 2) Update counters. - * 3) Increment epoch to an even number. - * - * The reader must assure 1) that the epoch is even while it reads the - * counters, and 2) that the epoch doesn't change between the time it - * starts and finishes reading the counters. - */ - unsigned epoch; - - /* Profiling counters. */ + /* Profiling counters, protected by tdata->lock. */ prof_cnt_t cnts; -}; -typedef rb_tree(prof_thr_cnt_t) prof_thr_cnt_tree_t; -struct prof_ctx_s { - /* Protects nlimbo, cnt_merged, and thr_cnts. */ + /* Associated global context. */ + prof_gctx_t *gctx; + + /* Linkage into gctx's tctxs. */ + rb_node(prof_tctx_t) tctx_link; + + /* Current dump-related state, protected by gctx->lock. */ + prof_tctx_state_t state; + + /* + * Copy of cnts snapshotted during early dump phase, protected by + * dump_mtx. + */ + prof_cnt_t dump_cnts; +}; +typedef rb_tree(prof_tctx_t) prof_tctx_tree_t; + +struct prof_gctx_s { + /* Protects nlimbo, cnt_summed, and tctxs. */ malloc_mutex_t *lock; /* - * Number of threads that currently cause this ctx to be in a state of + * Number of threads that currently cause this gctx to be in a state of * limbo due to one of: - * - Initializing per thread counters associated with this ctx. - * - Preparing to destroy this ctx. - * - Dumping a heap profile that includes this ctx. + * - Initializing this gctx. + * - Initializing per thread counters associated with this gctx. + * - Preparing to destroy this gctx. + * - Dumping a heap profile that includes this gctx. * nlimbo must be 1 (single destroyer) in order to safely destroy the - * ctx. + * gctx. */ unsigned nlimbo; - /* Temporary storage for summation during dump. */ - prof_cnt_t cnt_summed; - - /* When threads exit, they merge their stats into cnt_merged. */ - prof_cnt_t cnt_merged; - /* * Tree of profile counters, one for each thread that has allocated in * this context. */ - prof_thr_cnt_tree_t thr_cnts; + prof_tctx_tree_t tctxs; /* Linkage for tree of contexts to be dumped. */ - rb_node(prof_ctx_t) dump_link; + rb_node(prof_gctx_t) dump_link; + + /* Temporary storage for summation during dump. */ + prof_cnt_t cnt_summed; /* Associated backtrace. */ prof_bt_t bt; @@ -154,21 +142,34 @@ struct prof_ctx_s { /* Backtrace vector, variable size, referred to by bt. */ void *vec[1]; }; -typedef rb_tree(prof_ctx_t) prof_ctx_tree_t; +typedef rb_tree(prof_gctx_t) prof_gctx_tree_t; + +typedef enum { + prof_tdata_state_attached, /* Active thread attached, data valid. */ + prof_tdata_state_detached, /* Defunct thread, data remain valid. */ + prof_tdata_state_expired /* Predates reset, omit data from dump. */ +} prof_tdata_state_t; struct prof_tdata_s { + malloc_mutex_t *lock; + + /* Monotonically increasing unique thread identifier. */ + uint64_t thr_uid; + + /* Included in heap profile dumps if non-NULL. */ + char *thread_name; + + prof_tdata_state_t state; + + rb_node(prof_tdata_t) tdata_link; + /* - * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread tracks + * Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks * backtraces for which it has non-zero allocation/deallocation counters - * associated with thread-specific prof_thr_cnt_t objects. Other - * threads may read the prof_thr_cnt_t contents, but no others will ever - * write them. - * - * Upon thread exit, the thread must merge all the prof_thr_cnt_t - * counter data into the associated prof_ctx_t objects, and unlink/free - * the prof_thr_cnt_t objects. + * associated with thread-specific prof_tctx_t objects. Other threads + * may write to prof_tctx_t contents when freeing associated objects. */ - ckh_t bt2cnt; + ckh_t bt2tctx; /* Sampling state. */ uint64_t prng_state; @@ -179,9 +180,27 @@ struct prof_tdata_s { bool enq_idump; bool enq_gdump; + /* + * Set to true during an early dump phase for tdata's which are + * currently being dumped. New threads' tdata's have this initialized + * to false so that they aren't accidentally included in later dump + * phases. + */ + bool dumping; + + /* + * True if profiling is active for this tdata's thread + * (thread.prof.active mallctl). + */ + bool active; + + /* Temporary storage for summation during dump. */ + prof_cnt_t cnt_summed; + /* Backtrace vector, used for calls to prof_backtrace(). */ void *vec[PROF_BT_MAX]; }; +typedef rb_tree(prof_tdata_t) prof_tdata_tree_t; #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ @@ -217,9 +236,18 @@ extern char opt_prof_prefix[ */ extern uint64_t prof_interval; +/* + * Initialized as opt_lg_prof_sample, and potentially modified during profiling + * resets. + */ +extern size_t lg_prof_sample; + +void prof_malloc_sample_object(const void *ptr, size_t usize, + prof_tctx_t *tctx); +void prof_free_sampled_object(size_t usize, prof_tctx_t *tctx); void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt); -prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); +prof_tctx_t *prof_lookup(prof_bt_t *bt); #ifdef JEMALLOC_JET size_t prof_bt_count(void); typedef int (prof_dump_open_t)(bool, const char *); @@ -229,53 +257,44 @@ void prof_idump(void); bool prof_mdump(const char *filename); void prof_gdump(void); prof_tdata_t *prof_tdata_init(void); +prof_tdata_t *prof_tdata_reinit(prof_tdata_t *tdata); +void prof_reset(size_t lg_sample); void prof_tdata_cleanup(void *arg); +const char *prof_thread_name_get(void); +bool prof_thread_name_set(const char *thread_name); +bool prof_thread_active_get(void); +bool prof_thread_active_set(bool active); void prof_boot0(void); void prof_boot1(void); bool prof_boot2(void); void prof_prefork(void); void prof_postfork_parent(void); void prof_postfork_child(void); -void prof_sample_threshold_update(prof_tdata_t *prof_tdata); +void prof_sample_threshold_update(prof_tdata_t *tdata); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES -#define PROF_ALLOC_PREP(size, ret) do { \ - prof_tdata_t *prof_tdata; \ - prof_bt_t bt; \ - \ - assert(size == s2u(size)); \ - \ - if (!opt_prof_active || \ - prof_sample_accum_update(size, false, &prof_tdata)) { \ - ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ - } else { \ - bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt); \ - ret = prof_lookup(&bt); \ - } \ -} while (0) - #ifndef JEMALLOC_ENABLE_INLINE malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) prof_tdata_t *prof_tdata_get(bool create); -bool prof_sample_accum_update(size_t size, bool commit, - prof_tdata_t **prof_tdata_out); -prof_ctx_t *prof_ctx_get(const void *ptr); -void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -void prof_malloc_record_object(const void *ptr, size_t usize, - prof_thr_cnt_t *cnt); -void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); -void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, - size_t old_usize, prof_ctx_t *old_ctx); -void prof_free(const void *ptr, size_t size); +bool prof_sample_accum_update(size_t usize, bool commit, + prof_tdata_t **tdata_out); +prof_tctx_t *prof_alloc_prep(size_t usize); +prof_tctx_t *prof_tctx_get(const void *ptr); +void prof_tctx_set(const void *ptr, prof_tctx_t *tctx); +void prof_malloc_sample_object(const void *ptr, size_t usize, + prof_tctx_t *tctx); +void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx); +void prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, + size_t old_usize, prof_tctx_t *old_tctx); +void prof_free(const void *ptr, size_t usize); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) -/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ +/* Thread-specific backtrace cache, used to reduce bt2gctx contention. */ malloc_tsd_externs(prof_tdata, prof_tdata_t *) malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, prof_tdata_cleanup) @@ -283,21 +302,27 @@ malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, JEMALLOC_INLINE prof_tdata_t * prof_tdata_get(bool create) { - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; cassert(config_prof); - prof_tdata = *prof_tdata_tsd_get(); - if (create && prof_tdata == NULL) - prof_tdata = prof_tdata_init(); + tdata = *prof_tdata_tsd_get(); + if (create) { + if (tdata == NULL) + tdata = prof_tdata_init(); + else if (tdata->state == prof_tdata_state_expired) + tdata = prof_tdata_reinit(tdata); + assert(tdata == NULL || tdata->state == + prof_tdata_state_attached); + } - return (prof_tdata); + return (tdata); } -JEMALLOC_INLINE prof_ctx_t * -prof_ctx_get(const void *ptr) +JEMALLOC_INLINE prof_tctx_t * +prof_tctx_get(const void *ptr) { - prof_ctx_t *ret; + prof_tctx_t *ret; arena_chunk_t *chunk; cassert(config_prof); @@ -306,15 +331,15 @@ prof_ctx_get(const void *ptr) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - ret = arena_prof_ctx_get(ptr); + ret = arena_prof_tctx_get(ptr); } else - ret = huge_prof_ctx_get(ptr); + ret = huge_prof_tctx_get(ptr); return (ret); } JEMALLOC_INLINE void -prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { arena_chunk_t *chunk; @@ -324,66 +349,62 @@ prof_ctx_set(const void *ptr, prof_ctx_t *ctx) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - arena_prof_ctx_set(ptr, ctx); + arena_prof_tctx_set(ptr, tctx); } else - huge_prof_ctx_set(ptr, ctx); + huge_prof_tctx_set(ptr, tctx); } JEMALLOC_INLINE bool -prof_sample_accum_update(size_t size, bool commit, - prof_tdata_t **prof_tdata_out) +prof_sample_accum_update(size_t usize, bool commit, prof_tdata_t **tdata_out) { - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; cassert(config_prof); - prof_tdata = prof_tdata_get(true); - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) - prof_tdata = NULL; + tdata = prof_tdata_get(true); + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = NULL; - if (prof_tdata_out != NULL) - *prof_tdata_out = prof_tdata; + if (tdata_out != NULL) + *tdata_out = tdata; - if (prof_tdata == NULL) + if (tdata == NULL) return (true); - if (prof_tdata->bytes_until_sample >= size) { + if (tdata->bytes_until_sample >= usize) { if (commit) - prof_tdata->bytes_until_sample -= size; + tdata->bytes_until_sample -= usize; return (true); } else { /* Compute new sample threshold. */ if (commit) - prof_sample_threshold_update(prof_tdata); - return (false); + prof_sample_threshold_update(tdata); + return (tdata->active == false); } } -JEMALLOC_INLINE void -prof_malloc_record_object(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) { - prof_ctx_set(ptr, cnt->ctx); +JEMALLOC_INLINE prof_tctx_t * +prof_alloc_prep(size_t usize) +{ + prof_tctx_t *ret; + prof_tdata_t *tdata; + prof_bt_t bt; - cnt->epoch++; - /*********/ - mb_write(); - /*********/ - cnt->cnts.curobjs++; - cnt->cnts.curbytes += usize; - if (opt_prof_accum) { - cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += usize; + assert(usize == s2u(usize)); + + if (!opt_prof_active || prof_sample_accum_update(usize, false, &tdata)) + ret = (prof_tctx_t *)(uintptr_t)1U; + else { + bt_init(&bt, tdata->vec); + prof_backtrace(&bt); + ret = prof_lookup(&bt); } - /*********/ - mb_write(); - /*********/ - cnt->epoch++; - /*********/ - mb_write(); - /*********/ + + return (ret); } JEMALLOC_INLINE void -prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) +prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) { cassert(config_prof); @@ -392,131 +413,60 @@ prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) if (prof_sample_accum_update(usize, true, NULL)) { /* - * Don't sample. For malloc()-like allocation, it is - * always possible to tell in advance how large an - * object's usable size will be, so there should never - * be a difference between the usize passed to - * PROF_ALLOC_PREP() and prof_malloc(). + * Don't sample. For malloc()-like allocation, it is always + * possible to tell in advance how large an object's usable size + * will be, so there should never be a difference between the + * usize passed to PROF_ALLOC_PREP() and prof_malloc(). */ - assert((uintptr_t)cnt == (uintptr_t)1U); + assert((uintptr_t)tctx == (uintptr_t)1U); } - if ((uintptr_t)cnt > (uintptr_t)1U) - prof_malloc_record_object(ptr, usize, cnt); + if ((uintptr_t)tctx > (uintptr_t)1U) + prof_malloc_sample_object(ptr, usize, tctx); else - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); + prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); } JEMALLOC_INLINE void -prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, - size_t old_usize, prof_ctx_t *old_ctx) +prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, size_t old_usize, + prof_tctx_t *old_tctx) { - prof_thr_cnt_t *told_cnt; cassert(config_prof); - assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); + assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); if (ptr != NULL) { assert(usize == isalloc(ptr, true)); if (prof_sample_accum_update(usize, true, NULL)) { /* - * Don't sample. The usize passed to - * PROF_ALLOC_PREP() was larger than what - * actually got allocated, so a backtrace was - * captured for this allocation, even though - * its actual usize was insufficient to cross - * the sample threshold. + * Don't sample. The usize passed to PROF_ALLOC_PREP() + * was larger than what actually got allocated, so a + * backtrace was captured for this allocation, even + * though its actual usize was insufficient to cross the + * sample threshold. */ - cnt = (prof_thr_cnt_t *)(uintptr_t)1U; + tctx = (prof_tctx_t *)(uintptr_t)1U; } } - if ((uintptr_t)old_ctx > (uintptr_t)1U) { - told_cnt = prof_lookup(&old_ctx->bt); - if (told_cnt == NULL) { - /* - * It's too late to propagate OOM for this realloc(), - * so operate directly on old_cnt->ctx->cnt_merged. - */ - malloc_mutex_lock(old_ctx->lock); - old_ctx->cnt_merged.curobjs--; - old_ctx->cnt_merged.curbytes -= old_usize; - malloc_mutex_unlock(old_ctx->lock); - told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; - } - } else - told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; - - if ((uintptr_t)told_cnt > (uintptr_t)1U) - told_cnt->epoch++; - if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); - cnt->epoch++; - } else if (ptr != NULL) - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); - /*********/ - mb_write(); - /*********/ - if ((uintptr_t)told_cnt > (uintptr_t)1U) { - told_cnt->cnts.curobjs--; - told_cnt->cnts.curbytes -= old_usize; - } - if ((uintptr_t)cnt > (uintptr_t)1U) { - cnt->cnts.curobjs++; - cnt->cnts.curbytes += usize; - if (opt_prof_accum) { - cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += usize; - } - } - /*********/ - mb_write(); - /*********/ - if ((uintptr_t)told_cnt > (uintptr_t)1U) - told_cnt->epoch++; - if ((uintptr_t)cnt > (uintptr_t)1U) - cnt->epoch++; - /*********/ - mb_write(); /* Not strictly necessary. */ + if ((uintptr_t)old_tctx > (uintptr_t)1U) + prof_free_sampled_object(old_usize, old_tctx); + if ((uintptr_t)tctx > (uintptr_t)1U) + prof_malloc_sample_object(ptr, usize, tctx); + else + prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); } JEMALLOC_INLINE void -prof_free(const void *ptr, size_t size) +prof_free(const void *ptr, size_t usize) { - prof_ctx_t *ctx = prof_ctx_get(ptr); + prof_tctx_t *tctx = prof_tctx_get(ptr); cassert(config_prof); + assert(usize == isalloc(ptr, true)); - if ((uintptr_t)ctx > (uintptr_t)1) { - prof_thr_cnt_t *tcnt; - assert(size == isalloc(ptr, true)); - tcnt = prof_lookup(&ctx->bt); - - if (tcnt != NULL) { - tcnt->epoch++; - /*********/ - mb_write(); - /*********/ - tcnt->cnts.curobjs--; - tcnt->cnts.curbytes -= size; - /*********/ - mb_write(); - /*********/ - tcnt->epoch++; - /*********/ - mb_write(); - /*********/ - } else { - /* - * OOM during free() cannot be propagated, so operate - * directly on cnt->ctx->cnt_merged. - */ - malloc_mutex_lock(ctx->lock); - ctx->cnt_merged.curobjs--; - ctx->cnt_merged.curbytes -= size; - malloc_mutex_unlock(ctx->lock); - } - } + if ((uintptr_t)tctx > (uintptr_t)1U) + prof_free_sampled_object(usize, tctx); } #endif diff --git a/src/ctl.c b/src/ctl.c index fa52a6cc..b816c845 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -68,6 +68,8 @@ CTL_PROTO(version) CTL_PROTO(epoch) CTL_PROTO(thread_tcache_enabled) CTL_PROTO(thread_tcache_flush) +CTL_PROTO(thread_prof_name) +CTL_PROTO(thread_prof_active) CTL_PROTO(thread_arena) CTL_PROTO(thread_allocated) CTL_PROTO(thread_allocatedp) @@ -132,7 +134,9 @@ CTL_PROTO(arenas_nlruns) CTL_PROTO(arenas_extend) CTL_PROTO(prof_active) CTL_PROTO(prof_dump) +CTL_PROTO(prof_reset) CTL_PROTO(prof_interval) +CTL_PROTO(lg_prof_sample) CTL_PROTO(stats_chunks_current) CTL_PROTO(stats_chunks_total) CTL_PROTO(stats_chunks_high) @@ -196,18 +200,24 @@ CTL_PROTO(stats_mapped) */ #define INDEX(i) {false}, i##_index -static const ctl_named_node_t tcache_node[] = { +static const ctl_named_node_t thread_tcache_node[] = { {NAME("enabled"), CTL(thread_tcache_enabled)}, {NAME("flush"), CTL(thread_tcache_flush)} }; +static const ctl_named_node_t thread_prof_node[] = { + {NAME("name"), CTL(thread_prof_name)}, + {NAME("active"), CTL(thread_prof_active)} +}; + static const ctl_named_node_t thread_node[] = { {NAME("arena"), CTL(thread_arena)}, {NAME("allocated"), CTL(thread_allocated)}, {NAME("allocatedp"), CTL(thread_allocatedp)}, {NAME("deallocated"), CTL(thread_deallocated)}, {NAME("deallocatedp"), CTL(thread_deallocatedp)}, - {NAME("tcache"), CHILD(named, tcache)} + {NAME("tcache"), CHILD(named, thread_tcache)}, + {NAME("prof"), CHILD(named, thread_prof)} }; static const ctl_named_node_t config_node[] = { @@ -311,7 +321,9 @@ static const ctl_named_node_t arenas_node[] = { static const ctl_named_node_t prof_node[] = { {NAME("active"), CTL(prof_active)}, {NAME("dump"), CTL(prof_dump)}, - {NAME("interval"), CTL(prof_interval)} + {NAME("reset"), CTL(prof_reset)}, + {NAME("interval"), CTL(prof_interval)}, + {NAME("lg_sample"), CTL(lg_prof_sample)} }; static const ctl_named_node_t stats_chunks_node[] = { @@ -1281,6 +1293,62 @@ label_return: return (ret); } +static int +thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + const char *oldname; + + if (config_prof == false) + return (ENOENT); + + oldname = prof_thread_name_get(); + if (newp != NULL) { + if (newlen != sizeof(const char *)) { + ret = EINVAL; + goto label_return; + } + if (prof_thread_name_set(*(const char **)newp)) { + ret = EAGAIN; + goto label_return; + } + } + READ(oldname, const char *); + + ret = 0; +label_return: + return (ret); +} + +static int +thread_prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + bool oldval; + + if (config_prof == false) + return (ENOENT); + + oldval = prof_thread_active_get(); + if (newp != NULL) { + if (newlen != sizeof(bool)) { + ret = EINVAL; + goto label_return; + } + if (prof_thread_active_set(*(bool *)newp)) { + ret = EAGAIN; + goto label_return; + } + } + READ(oldval, bool); + + ret = 0; +label_return: + return (ret); +} + /******************************************************************************/ /* ctl_mutex must be held during execution of this function. */ @@ -1601,7 +1669,30 @@ label_return: return (ret); } +static int +prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + size_t lg_sample = lg_prof_sample; + + if (config_prof == false) + return (ENOENT); + + WRITEONLY(); + WRITE(lg_sample, size_t); + if (lg_sample >= (sizeof(uint64_t) << 3)) + lg_sample = (sizeof(uint64_t) << 3) - 1; + + prof_reset(lg_sample); + + ret = 0; +label_return: + return (ret); +} + CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) +CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) /******************************************************************************/ diff --git a/src/huge.c b/src/huge.c index d08ed4a9..5f0c6980 100644 --- a/src/huge.c +++ b/src/huge.c @@ -197,10 +197,10 @@ huge_salloc(const void *ptr) return (ret); } -prof_ctx_t * -huge_prof_ctx_get(const void *ptr) +prof_tctx_t * +huge_prof_tctx_get(const void *ptr) { - prof_ctx_t *ret; + prof_tctx_t *ret; extent_node_t *node, key; malloc_mutex_lock(&huge_mtx); @@ -210,7 +210,7 @@ huge_prof_ctx_get(const void *ptr) node = extent_tree_ad_search(&huge, &key); assert(node != NULL); - ret = node->prof_ctx; + ret = node->prof_tctx; malloc_mutex_unlock(&huge_mtx); @@ -218,7 +218,7 @@ huge_prof_ctx_get(const void *ptr) } void -huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { extent_node_t *node, key; @@ -229,7 +229,7 @@ huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) node = extent_tree_ad_search(&huge, &key); assert(node != NULL); - node->prof_ctx = ctx; + node->prof_tctx = tctx; malloc_mutex_unlock(&huge_mtx); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 0983c00d..2d01272e 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -636,9 +636,9 @@ malloc_conf_init(void) "prof_prefix", "jeprof") CONF_HANDLE_BOOL(opt_prof_active, "prof_active", true) - CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, + CONF_HANDLE_SIZE_T(opt_lg_prof_sample, "lg_prof_sample", 0, - (sizeof(uint64_t) << 3) - 1) + (sizeof(uint64_t) << 3) - 1, true) CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum", true) CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, @@ -863,11 +863,11 @@ malloc_init_hard(void) */ static void * -imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) +imalloc_prof_sample(size_t usize, prof_tctx_t *tctx) { void *p; - if (cnt == NULL) + if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { p = imalloc(SMALL_MAXCLASS+1); @@ -884,16 +884,16 @@ JEMALLOC_ALWAYS_INLINE_C void * imalloc_prof(size_t usize) { void *p; - prof_thr_cnt_t *cnt; + prof_tctx_t *tctx; - PROF_ALLOC_PREP(usize, cnt); - if ((uintptr_t)cnt != (uintptr_t)1U) - p = imalloc_prof_sample(usize, cnt); + tctx = prof_alloc_prep(usize); + if ((uintptr_t)tctx != (uintptr_t)1U) + p = imalloc_prof_sample(usize, tctx); else p = imalloc(usize); if (p == NULL) return (NULL); - prof_malloc(p, usize, cnt); + prof_malloc(p, usize, tctx); return (p); } @@ -943,11 +943,11 @@ je_malloc(size_t size) } static void * -imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) +imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx) { void *p; - if (cnt == NULL) + if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0); @@ -963,17 +963,17 @@ imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) } JEMALLOC_ALWAYS_INLINE_C void * -imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) +imemalign_prof(size_t alignment, size_t usize, prof_tctx_t *tctx) { void *p; - if ((uintptr_t)cnt != (uintptr_t)1U) - p = imemalign_prof_sample(alignment, usize, cnt); + if ((uintptr_t)tctx != (uintptr_t)1U) + p = imemalign_prof_sample(alignment, usize, tctx); else p = ipalloc(usize, alignment, false); if (p == NULL) return (NULL); - prof_malloc(p, usize, cnt); + prof_malloc(p, usize, tctx); return (p); } @@ -1015,10 +1015,10 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) } if (config_prof && opt_prof) { - prof_thr_cnt_t *cnt; + prof_tctx_t *tctx; - PROF_ALLOC_PREP(usize, cnt); - result = imemalign_prof(alignment, usize, cnt); + tctx = prof_alloc_prep(usize); + result = imemalign_prof(alignment, usize, tctx); } else result = ipalloc(usize, alignment, false); if (result == NULL) @@ -1070,11 +1070,11 @@ je_aligned_alloc(size_t alignment, size_t size) } static void * -icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) +icalloc_prof_sample(size_t usize, prof_tctx_t *tctx) { void *p; - if (cnt == NULL) + if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { p = icalloc(SMALL_MAXCLASS+1); @@ -1088,17 +1088,17 @@ icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) } JEMALLOC_ALWAYS_INLINE_C void * -icalloc_prof(size_t usize, prof_thr_cnt_t *cnt) +icalloc_prof(size_t usize, prof_tctx_t *tctx) { void *p; - if ((uintptr_t)cnt != (uintptr_t)1U) - p = icalloc_prof_sample(usize, cnt); + if ((uintptr_t)tctx != (uintptr_t)1U) + p = icalloc_prof_sample(usize, tctx); else p = icalloc(usize); if (p == NULL) return (NULL); - prof_malloc(p, usize, cnt); + prof_malloc(p, usize, tctx); return (p); } @@ -1137,11 +1137,11 @@ je_calloc(size_t num, size_t size) } if (config_prof && opt_prof) { - prof_thr_cnt_t *cnt; + prof_tctx_t *tctx; usize = s2u(num_size); - PROF_ALLOC_PREP(usize, cnt); - ret = icalloc_prof(usize, cnt); + tctx = prof_alloc_prep(usize); + ret = icalloc_prof(usize, tctx); } else { if (config_stats || (config_valgrind && in_valgrind)) usize = s2u(num_size); @@ -1167,11 +1167,11 @@ label_return: } static void * -irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) +irealloc_prof_sample(void *oldptr, size_t usize, prof_tctx_t *tctx) { void *p; - if (cnt == NULL) + if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false); @@ -1185,19 +1185,19 @@ irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) } JEMALLOC_ALWAYS_INLINE_C void * -irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt) +irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_tctx_t *tctx) { void *p; - prof_ctx_t *old_ctx; + prof_tctx_t *old_tctx; - old_ctx = prof_ctx_get(oldptr); - if ((uintptr_t)cnt != (uintptr_t)1U) - p = irealloc_prof_sample(oldptr, usize, cnt); + old_tctx = prof_tctx_get(oldptr); + if ((uintptr_t)tctx != (uintptr_t)1U) + p = irealloc_prof_sample(oldptr, usize, tctx); else p = iralloc(oldptr, usize, 0, 0, false); if (p == NULL) return (NULL); - prof_realloc(p, usize, cnt, old_usize, old_ctx); + prof_realloc(p, usize, tctx, old_usize, old_tctx); return (p); } @@ -1253,11 +1253,11 @@ je_realloc(void *ptr, size_t size) old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); if (config_prof && opt_prof) { - prof_thr_cnt_t *cnt; + prof_tctx_t *tctx; usize = s2u(size); - PROF_ALLOC_PREP(usize, cnt); - ret = irealloc_prof(ptr, old_usize, usize, cnt); + tctx = prof_alloc_prep(usize); + ret = irealloc_prof(ptr, old_usize, usize, tctx); } else { if (config_stats || (config_valgrind && in_valgrind)) usize = s2u(size); @@ -1379,11 +1379,11 @@ imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, static void * imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena, prof_thr_cnt_t *cnt) + arena_t *arena, prof_tctx_t *tctx) { void *p; - if (cnt == NULL) + if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { size_t usize_promoted = (alignment == 0) ? @@ -1402,18 +1402,18 @@ imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, JEMALLOC_ALWAYS_INLINE_C void * imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena, prof_thr_cnt_t *cnt) + arena_t *arena, prof_tctx_t *tctx) { void *p; - if ((uintptr_t)cnt != (uintptr_t)1U) { + if ((uintptr_t)tctx != (uintptr_t)1U) { p = imallocx_prof_sample(usize, alignment, zero, try_tcache, - arena, cnt); + arena, tctx); } else p = imallocx(usize, alignment, zero, try_tcache, arena); if (p == NULL) return (NULL); - prof_malloc(p, usize, cnt); + prof_malloc(p, usize, tctx); return (p); } @@ -1447,11 +1447,11 @@ je_mallocx(size_t size, int flags) assert(usize != 0); if (config_prof && opt_prof) { - prof_thr_cnt_t *cnt; + prof_tctx_t *tctx; - PROF_ALLOC_PREP(usize, cnt); + tctx = prof_alloc_prep(usize); p = imallocx_prof(usize, alignment, zero, try_tcache, arena, - cnt); + tctx); } else p = imallocx(usize, alignment, zero, try_tcache, arena); if (p == NULL) @@ -1476,11 +1476,11 @@ label_oom: static void * irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena, - prof_thr_cnt_t *cnt) + prof_tctx_t *tctx) { void *p; - if (cnt == NULL) + if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= @@ -1500,15 +1500,15 @@ irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, JEMALLOC_ALWAYS_INLINE_C void * irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, - arena_t *arena, prof_thr_cnt_t *cnt) + arena_t *arena, prof_tctx_t *tctx) { void *p; - prof_ctx_t *old_ctx; + prof_tctx_t *old_tctx; - old_ctx = prof_ctx_get(oldptr); - if ((uintptr_t)cnt != (uintptr_t)1U) + old_tctx = prof_tctx_get(oldptr); + if ((uintptr_t)tctx != (uintptr_t)1U) p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, - try_tcache_alloc, try_tcache_dalloc, arena, cnt); + try_tcache_alloc, try_tcache_dalloc, arena, tctx); else { p = iralloct(oldptr, size, 0, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); @@ -1527,7 +1527,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, */ *usize = isalloc(p, config_prof); } - prof_realloc(p, *usize, cnt, old_usize, old_ctx); + prof_realloc(p, *usize, tctx, old_usize, old_tctx); return (p); } @@ -1570,13 +1570,13 @@ je_rallocx(void *ptr, size_t size, int flags) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { - prof_thr_cnt_t *cnt; + prof_tctx_t *tctx; usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); assert(usize != 0); - PROF_ALLOC_PREP(usize, cnt); + tctx = prof_alloc_prep(usize); p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, - try_tcache_alloc, try_tcache_dalloc, arena, cnt); + try_tcache_alloc, try_tcache_dalloc, arena, tctx); if (p == NULL) goto label_oom; } else { @@ -1623,11 +1623,11 @@ ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, static size_t ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, size_t max_usize, bool zero, arena_t *arena, - prof_thr_cnt_t *cnt) + prof_tctx_t *tctx) { size_t usize; - if (cnt == NULL) + if (tctx == NULL) return (old_usize); /* Use minimum usize to determine whether promotion may happen. */ if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= @@ -1650,22 +1650,22 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, JEMALLOC_ALWAYS_INLINE_C size_t ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, size_t max_usize, bool zero, arena_t *arena, - prof_thr_cnt_t *cnt) + prof_tctx_t *tctx) { size_t usize; - prof_ctx_t *old_ctx; + prof_tctx_t *old_tctx; - old_ctx = prof_ctx_get(ptr); - if ((uintptr_t)cnt != (uintptr_t)1U) { + old_tctx = prof_tctx_get(ptr); + if ((uintptr_t)tctx != (uintptr_t)1U) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, - alignment, zero, max_usize, arena, cnt); + alignment, zero, max_usize, arena, tctx); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero, arena); } if (usize == old_usize) return (usize); - prof_realloc(ptr, usize, cnt, old_usize, old_ctx); + prof_realloc(ptr, usize, tctx, old_usize, old_tctx); return (usize); } @@ -1697,19 +1697,19 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { - prof_thr_cnt_t *cnt; + prof_tctx_t *tctx; /* * usize isn't knowable before ixalloc() returns when extra is * non-zero. Therefore, compute its maximum possible value and - * use that in PROF_ALLOC_PREP() to decide whether to capture a + * use that in prof_alloc_prep() to decide whether to capture a * backtrace. prof_realloc() will use the actual usize to * decide whether to sample. */ size_t max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment); - PROF_ALLOC_PREP(max_usize, cnt); + tctx = prof_alloc_prep(max_usize); usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, - max_usize, zero, arena, cnt); + max_usize, zero, arena, tctx); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero, arena); diff --git a/src/prof.c b/src/prof.c index 497ccf42..044acd8b 100644 --- a/src/prof.c +++ b/src/prof.c @@ -33,22 +33,41 @@ char opt_prof_prefix[ uint64_t prof_interval = 0; +size_t lg_prof_sample; + /* - * Table of mutexes that are shared among ctx's. These are leaf locks, so - * there is no problem with using them for more than one ctx at the same time. - * The primary motivation for this sharing though is that ctx's are ephemeral, + * Table of mutexes that are shared among gctx's. These are leaf locks, so + * there is no problem with using them for more than one gctx at the same time. + * The primary motivation for this sharing though is that gctx's are ephemeral, * and destroying mutexes causes complications for systems that allocate when * creating/destroying mutexes. */ -static malloc_mutex_t *ctx_locks; -static unsigned cum_ctxs; /* Atomic counter. */ +static malloc_mutex_t *gctx_locks; +static unsigned cum_gctxs; /* Atomic counter. */ /* - * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data + * Table of mutexes that are shared among tdata's. No operations require + * holding multiple tdata locks, so there is no problem with using them for more + * than one tdata at the same time, even though a gctx lock may be acquired + * while holding a tdata lock. + */ +static malloc_mutex_t *tdata_locks; + +/* + * Global hash of (prof_bt_t *)-->(prof_gctx_t *). This is the master data * structure that knows about all backtraces currently captured. */ -static ckh_t bt2ctx; -static malloc_mutex_t bt2ctx_mtx; +static ckh_t bt2gctx; +static malloc_mutex_t bt2gctx_mtx; + +/* + * Tree of all extant prof_tdata_t structures, regardless of state, + * {attached,detached,expired}. + */ +static prof_tdata_tree_t tdatas; +static malloc_mutex_t tdatas_mtx; + +static uint64_t next_thr_uid; static malloc_mutex_t prof_dump_seq_mtx; static uint64_t prof_dump_seq; @@ -76,21 +95,33 @@ static int prof_dump_fd; static bool prof_booted = false; /******************************************************************************/ +/* + * Function prototypes for static functions that are referenced prior to + * definition. + */ + +static bool prof_tctx_should_destroy(prof_tctx_t *tctx); +static void prof_tctx_destroy(prof_tctx_t *tctx); +static bool prof_tdata_should_destroy(prof_tdata_t *tdata); +static void prof_tdata_destroy(prof_tdata_t *tdata); + +/******************************************************************************/ +/* Red-black trees. */ JEMALLOC_INLINE_C int -prof_thr_cnt_comp(const prof_thr_cnt_t *a, const prof_thr_cnt_t *b) +prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) { - prof_thr_uid_t a_uid = a->thr_uid; - prof_thr_uid_t b_uid = b->thr_uid; + uint64_t a_uid = a->tdata->thr_uid; + uint64_t b_uid = b->tdata->thr_uid; return ((a_uid > b_uid) - (a_uid < b_uid)); } -rb_gen(static UNUSED, thr_cnt_tree_, prof_thr_cnt_tree_t, prof_thr_cnt_t, - thr_cnt_link, prof_thr_cnt_comp) +rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, + tctx_link, prof_tctx_comp) JEMALLOC_INLINE_C int -prof_ctx_comp(const prof_ctx_t *a, const prof_ctx_t *b) +prof_gctx_comp(const prof_gctx_t *a, const prof_gctx_t *b) { unsigned a_len = a->bt.len; unsigned b_len = b->bt.len; @@ -101,8 +132,52 @@ prof_ctx_comp(const prof_ctx_t *a, const prof_ctx_t *b) return (ret); } -rb_gen(static UNUSED, ctx_tree_, prof_ctx_tree_t, prof_ctx_t, dump_link, - prof_ctx_comp) +rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, + prof_gctx_comp) + +JEMALLOC_INLINE_C int +prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) +{ + uint64_t a_uid = a->thr_uid; + uint64_t b_uid = b->thr_uid; + + return ((a_uid > b_uid) - (a_uid < b_uid)); +} + +rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, + prof_tdata_comp) + +/******************************************************************************/ + +void +prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) { + prof_tctx_set(ptr, tctx); + + malloc_mutex_lock(tctx->tdata->lock); + tctx->cnts.curobjs++; + tctx->cnts.curbytes += usize; + if (opt_prof_accum) { + tctx->cnts.accumobjs++; + tctx->cnts.accumbytes += usize; + } + malloc_mutex_unlock(tctx->tdata->lock); +} + +void +prof_free_sampled_object(size_t usize, prof_tctx_t *tctx) +{ + + malloc_mutex_lock(tctx->tdata->lock); + assert(tctx->cnts.curobjs > 0); + assert(tctx->cnts.curbytes >= usize); + tctx->cnts.curobjs--; + tctx->cnts.curbytes -= usize; + + if (prof_tctx_should_destroy(tctx)) + prof_tctx_destroy(tctx); + else + malloc_mutex_unlock(tctx->tdata->lock); +} void bt_init(prof_bt_t *bt, void **vec) @@ -115,32 +190,32 @@ bt_init(prof_bt_t *bt, void **vec) } static inline void -prof_enter(prof_tdata_t *prof_tdata) +prof_enter(prof_tdata_t *tdata) { cassert(config_prof); - assert(prof_tdata->enq == false); - prof_tdata->enq = true; + assert(tdata->enq == false); + tdata->enq = true; - malloc_mutex_lock(&bt2ctx_mtx); + malloc_mutex_lock(&bt2gctx_mtx); } static inline void -prof_leave(prof_tdata_t *prof_tdata) +prof_leave(prof_tdata_t *tdata) { bool idump, gdump; cassert(config_prof); - malloc_mutex_unlock(&bt2ctx_mtx); + malloc_mutex_unlock(&bt2gctx_mtx); - assert(prof_tdata->enq); - prof_tdata->enq = false; - idump = prof_tdata->enq_idump; - prof_tdata->enq_idump = false; - gdump = prof_tdata->enq_gdump; - prof_tdata->enq_gdump = false; + assert(tdata->enq); + tdata->enq = false; + idump = tdata->enq_idump; + tdata->enq_idump = false; + gdump = tdata->enq_gdump; + tdata->enq_gdump = false; if (idump) prof_idump(); @@ -373,220 +448,268 @@ prof_backtrace(prof_bt_t *bt) #endif static malloc_mutex_t * -prof_ctx_mutex_choose(void) +prof_gctx_mutex_choose(void) { - unsigned nctxs = atomic_add_u(&cum_ctxs, 1); + unsigned ngctxs = atomic_add_u(&cum_gctxs, 1); - return (&ctx_locks[(nctxs - 1) % PROF_NCTX_LOCKS]); + return (&gctx_locks[(ngctxs - 1) % PROF_NCTX_LOCKS]); } -static prof_ctx_t * -prof_ctx_create(prof_bt_t *bt) +static malloc_mutex_t * +prof_tdata_mutex_choose(uint64_t thr_uid) +{ + + return (&tdata_locks[thr_uid % PROF_NTDATA_LOCKS]); +} + +static prof_gctx_t * +prof_gctx_create(prof_bt_t *bt) { /* * Create a single allocation that has space for vec of length bt->len. */ - prof_ctx_t *ctx = (prof_ctx_t *)imalloc(offsetof(prof_ctx_t, vec) + + prof_gctx_t *gctx = (prof_gctx_t *)imalloc(offsetof(prof_gctx_t, vec) + (bt->len * sizeof(void *))); - if (ctx == NULL) + if (gctx == NULL) return (NULL); - ctx->lock = prof_ctx_mutex_choose(); + gctx->lock = prof_gctx_mutex_choose(); /* * Set nlimbo to 1, in order to avoid a race condition with - * prof_ctx_merge()/prof_ctx_destroy(). + * prof_tctx_destroy()/prof_gctx_maybe_destroy(). */ - ctx->nlimbo = 1; - memset(&ctx->cnt_merged, 0, sizeof(prof_cnt_t)); - thr_cnt_tree_new(&ctx->thr_cnts); + gctx->nlimbo = 1; + tctx_tree_new(&gctx->tctxs); /* Duplicate bt. */ - memcpy(ctx->vec, bt->vec, bt->len * sizeof(void *)); - ctx->bt.vec = ctx->vec; - ctx->bt.len = bt->len; - return (ctx); + memcpy(gctx->vec, bt->vec, bt->len * sizeof(void *)); + gctx->bt.vec = gctx->vec; + gctx->bt.len = bt->len; + return (gctx); } static void -prof_ctx_destroy(prof_ctx_t *ctx) +prof_gctx_maybe_destroy(prof_gctx_t *gctx) { - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; cassert(config_prof); /* - * Check that ctx is still unused by any thread cache before destroying - * it. prof_lookup() increments ctx->nlimbo in order to avoid a race - * condition with this function, as does prof_ctx_merge() in order to - * avoid a race between the main body of prof_ctx_merge() and entry + * Check that gctx is still unused by any thread cache before destroying + * it. prof_lookup() increments gctx->nlimbo in order to avoid a race + * condition with this function, as does prof_tctx_destroy() in order to + * avoid a race between the main body of prof_tctx_destroy() and entry * into this function. */ - prof_tdata = prof_tdata_get(false); - assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX); - prof_enter(prof_tdata); - malloc_mutex_lock(ctx->lock); - if (thr_cnt_tree_first(&ctx->thr_cnts) == NULL && - ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 1) { - assert(ctx->cnt_merged.curbytes == 0); - assert(ctx->cnt_merged.accumobjs == 0); - assert(ctx->cnt_merged.accumbytes == 0); - /* Remove ctx from bt2ctx. */ - if (ckh_remove(&bt2ctx, &ctx->bt, NULL, NULL)) + tdata = prof_tdata_get(false); + assert((uintptr_t)tdata > (uintptr_t)PROF_TDATA_STATE_MAX); + prof_enter(tdata); + malloc_mutex_lock(gctx->lock); + if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { + /* Remove gctx from bt2gctx. */ + if (ckh_remove(&bt2gctx, &gctx->bt, NULL, NULL)) not_reached(); - prof_leave(prof_tdata); - /* Destroy ctx. */ - malloc_mutex_unlock(ctx->lock); - idalloc(ctx); + prof_leave(tdata); + /* Destroy gctx. */ + malloc_mutex_unlock(gctx->lock); + idalloc(gctx); } else { /* - * Compensate for increment in prof_ctx_merge() or + * Compensate for increment in prof_tctx_destroy() or * prof_lookup(). */ - ctx->nlimbo--; - malloc_mutex_unlock(ctx->lock); - prof_leave(prof_tdata); + gctx->nlimbo--; + malloc_mutex_unlock(gctx->lock); + prof_leave(tdata); } } -static void -prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) +/* tctx->tdata->lock must be held. */ +static bool +prof_tctx_should_destroy(prof_tctx_t *tctx) { - bool destroy; - cassert(config_prof); - - /* Merge cnt stats and detach from ctx. */ - malloc_mutex_lock(ctx->lock); - ctx->cnt_merged.curobjs += cnt->cnts.curobjs; - ctx->cnt_merged.curbytes += cnt->cnts.curbytes; - ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; - ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; - thr_cnt_tree_remove(&ctx->thr_cnts, cnt); - if (opt_prof_accum == false && thr_cnt_tree_first(&ctx->thr_cnts) == - NULL && ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) { - /* - * Increment ctx->nlimbo in order to keep another thread from - * winning the race to destroy ctx while this one has ctx->lock - * dropped. Without this, it would be possible for another - * thread to: - * - * 1) Sample an allocation associated with ctx. - * 2) Deallocate the sampled object. - * 3) Successfully prof_ctx_destroy(ctx). - * - * The result would be that ctx no longer exists by the time - * this thread accesses it in prof_ctx_destroy(). - */ - ctx->nlimbo++; - destroy = true; - } else - destroy = false; - malloc_mutex_unlock(ctx->lock); - if (destroy) - prof_ctx_destroy(ctx); + if (opt_prof_accum) + return (false); + if (tctx->cnts.curobjs != 0) + return (false); + return (true); } static bool -prof_lookup_global(prof_bt_t *bt, prof_tdata_t *prof_tdata, void **p_btkey, - prof_ctx_t **p_ctx, bool *p_new_ctx) +prof_gctx_should_destroy(prof_gctx_t *gctx) +{ + + if (opt_prof_accum) + return (false); + if (tctx_tree_empty(&gctx->tctxs) == false) + return (false); + if (gctx->nlimbo != 0) + return (false); + return (true); +} + +/* tctx->tdata->lock is held upon entry, and released before return. */ +static void +prof_tctx_destroy(prof_tctx_t *tctx) +{ + prof_gctx_t *gctx = tctx->gctx; + bool destroy_gctx; + + assert(tctx->cnts.curobjs == 0); + assert(tctx->cnts.curbytes == 0); + assert(opt_prof_accum == false); + assert(tctx->cnts.accumobjs == 0); + assert(tctx->cnts.accumbytes == 0); + + { + prof_tdata_t *tdata = tctx->tdata; + bool tdata_destroy; + + ckh_remove(&tdata->bt2tctx, &gctx->bt, NULL, NULL); + tdata_destroy = prof_tdata_should_destroy(tdata); + malloc_mutex_unlock(tdata->lock); + if (tdata_destroy) + prof_tdata_destroy(tdata); + } + + malloc_mutex_lock(gctx->lock); + tctx_tree_remove(&gctx->tctxs, tctx); + if (prof_gctx_should_destroy(gctx)) { + /* + * Increment gctx->nlimbo in order to keep another thread from + * winning the race to destroy gctx while this one has + * gctx->lock dropped. Without this, it would be possible for + * another thread to: + * + * 1) Sample an allocation associated with gctx. + * 2) Deallocate the sampled object. + * 3) Successfully prof_gctx_maybe_destroy(gctx). + * + * The result would be that gctx no longer exists by the time + * this thread accesses it in prof_gctx_maybe_destroy(). + */ + gctx->nlimbo++; + destroy_gctx = true; + } else + destroy_gctx = false; + malloc_mutex_unlock(gctx->lock); + if (destroy_gctx) + prof_gctx_maybe_destroy(gctx); + + idalloc(tctx); +} + +static bool +prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey, + prof_gctx_t **p_gctx, bool *p_new_gctx) { union { - prof_ctx_t *p; + prof_gctx_t *p; void *v; - } ctx; + } gctx; union { prof_bt_t *p; void *v; } btkey; - bool new_ctx; + bool new_gctx; - prof_enter(prof_tdata); - if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { + prof_enter(tdata); + if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { /* bt has never been seen before. Insert it. */ - ctx.p = prof_ctx_create(bt); - if (ctx.v == NULL) { - prof_leave(prof_tdata); + gctx.p = prof_gctx_create(bt); + if (gctx.v == NULL) { + prof_leave(tdata); return (true); } - btkey.p = &ctx.p->bt; - if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { + btkey.p = &gctx.p->bt; + if (ckh_insert(&bt2gctx, btkey.v, gctx.v)) { /* OOM. */ - prof_leave(prof_tdata); - idalloc(ctx.v); + prof_leave(tdata); + idalloc(gctx.v); return (true); } - new_ctx = true; + new_gctx = true; } else { /* * Increment nlimbo, in order to avoid a race condition with - * prof_ctx_merge()/prof_ctx_destroy(). + * prof_tctx_destroy()/prof_gctx_maybe_destroy(). */ - malloc_mutex_lock(ctx.p->lock); - ctx.p->nlimbo++; - malloc_mutex_unlock(ctx.p->lock); - new_ctx = false; + malloc_mutex_lock(gctx.p->lock); + gctx.p->nlimbo++; + malloc_mutex_unlock(gctx.p->lock); + new_gctx = false; } - prof_leave(prof_tdata); + prof_leave(tdata); *p_btkey = btkey.v; - *p_ctx = ctx.p; - *p_new_ctx = new_ctx; + *p_gctx = gctx.p; + *p_new_gctx = new_gctx; return (false); } -prof_thr_cnt_t * +prof_tctx_t * prof_lookup(prof_bt_t *bt) { union { - prof_thr_cnt_t *p; + prof_tctx_t *p; void *v; } ret; - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; + bool not_found; cassert(config_prof); - prof_tdata = prof_tdata_get(false); - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(false); + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (NULL); - if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { + malloc_mutex_lock(tdata->lock); + not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); + malloc_mutex_unlock(tdata->lock); + if (not_found) { void *btkey; - prof_ctx_t *ctx; - bool new_ctx; + prof_gctx_t *gctx; + bool new_gctx, error; /* * This thread's cache lacks bt. Look for it in the global * cache. */ - if (prof_lookup_global(bt, prof_tdata, &btkey, &ctx, &new_ctx)) + if (prof_lookup_global(bt, tdata, &btkey, &gctx, + &new_gctx)) return (NULL); - /* Link a prof_thd_cnt_t into ctx for this thread. */ - ret.v = imalloc(sizeof(prof_thr_cnt_t)); + /* Link a prof_tctx_t into gctx for this thread. */ + ret.v = imalloc(sizeof(prof_tctx_t)); if (ret.p == NULL) { - if (new_ctx) - prof_ctx_destroy(ctx); + if (new_gctx) + prof_gctx_maybe_destroy(gctx); return (NULL); } - ret.p->ctx = ctx; - ret.p->epoch = 0; + ret.p->tdata = tdata; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); - if (ckh_insert(&prof_tdata->bt2cnt, btkey, ret.v)) { - if (new_ctx) - prof_ctx_destroy(ctx); + ret.p->gctx = gctx; + ret.p->state = prof_tctx_state_nominal; + malloc_mutex_lock(tdata->lock); + error = ckh_insert(&tdata->bt2tctx, btkey, ret.v); + malloc_mutex_unlock(tdata->lock); + if (error) { + if (new_gctx) + prof_gctx_maybe_destroy(gctx); idalloc(ret.v); return (NULL); } - malloc_mutex_lock(ctx->lock); - thr_cnt_tree_insert(&ctx->thr_cnts, ret.p); - ctx->nlimbo--; - malloc_mutex_unlock(ctx->lock); + malloc_mutex_lock(gctx->lock); + tctx_tree_insert(&gctx->tctxs, ret.p); + gctx->nlimbo--; + malloc_mutex_unlock(gctx->lock); } return (ret.p); } - void -prof_sample_threshold_update(prof_tdata_t *prof_tdata) +prof_sample_threshold_update(prof_tdata_t *tdata) { /* * The body of this function is compiled out unless heap profiling is @@ -608,23 +731,20 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata) if (!config_prof) return; - if (prof_tdata == NULL) - prof_tdata = prof_tdata_get(false); - - if (opt_lg_prof_sample == 0) { - prof_tdata->bytes_until_sample = 0; + if (lg_prof_sample == 0) { + tdata->bytes_until_sample = 0; return; } /* - * Compute sample threshold as a geometrically distributed random - * variable with mean (2^opt_lg_prof_sample). + * Compute sample interval as a geometrically distributed random + * variable with mean (2^lg_prof_sample). * - * __ __ - * | log(u) | 1 - * prof_tdata->threshold = | -------- |, where p = ------------------- - * | log(1-p) | opt_lg_prof_sample - * 2 + * __ __ + * | log(u) | 1 + * tdata->bytes_until_sample = | -------- |, where p = --------------- + * | log(1-p) | lg_prof_sample + * 2 * * For more information on the math, see: * @@ -634,30 +754,29 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata) * pp 500 * (http://luc.devroye.org/rnbookindex.html) */ - prng64(r, 53, prof_tdata->prng_state, - UINT64_C(6364136223846793005), UINT64_C(1442695040888963407)); + prng64(r, 53, tdata->prng_state, UINT64_C(6364136223846793005), + UINT64_C(1442695040888963407)); u = (double)r * (1.0/9007199254740992.0L); - prof_tdata->bytes_until_sample = (uint64_t)(log(u) / - log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) + tdata->bytes_until_sample = (uint64_t)(log(u) / + log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) + (uint64_t)1U; #endif } - #ifdef JEMALLOC_JET size_t prof_bt_count(void) { size_t bt_count; - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; - prof_tdata = prof_tdata_get(false); - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(false); + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (0); - prof_enter(prof_tdata); - bt_count = ckh_count(&bt2ctx); - prof_leave(prof_tdata); + prof_enter(tdata); + bt_count = ckh_count(&bt2gctx); + prof_leave(tdata); return (bt_count); } @@ -770,146 +889,249 @@ prof_dump_printf(bool propagate_err, const char *format, ...) return (ret); } -static prof_thr_cnt_t * -ctx_sum_iter(prof_thr_cnt_tree_t *thr_cnts, prof_thr_cnt_t *thr_cnt, void *arg) +/* tctx->tdata->lock is held. */ +static void +prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata) { - prof_ctx_t *ctx = (prof_ctx_t *)arg; - volatile unsigned *epoch = &thr_cnt->epoch; - prof_cnt_t tcnt; - while (true) { - unsigned epoch0 = *epoch; + assert(tctx->state == prof_tctx_state_nominal); + tctx->state = prof_tctx_state_dumping; + memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); - /* Make sure epoch is even. */ - if (epoch0 & 1U) - continue; - - memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t)); - - /* Terminate if epoch didn't change while reading. */ - if (*epoch == epoch0) - break; - } - - ctx->cnt_summed.curobjs += tcnt.curobjs; - ctx->cnt_summed.curbytes += tcnt.curbytes; + tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; + tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; if (opt_prof_accum) { - ctx->cnt_summed.accumobjs += tcnt.accumobjs; - ctx->cnt_summed.accumbytes += tcnt.accumbytes; + tdata->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; + tdata->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; + } +} + +/* gctx->lock is held. */ +static void +prof_tctx_merge_gctx(prof_tctx_t *tctx, prof_gctx_t *gctx) +{ + + gctx->cnt_summed.curobjs += tctx->dump_cnts.curobjs; + gctx->cnt_summed.curbytes += tctx->dump_cnts.curbytes; + if (opt_prof_accum) { + gctx->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; + gctx->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; + } +} + +/* tctx->gctx is held. */ +static prof_tctx_t * +prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) +{ + + switch (tctx->state) { + case prof_tctx_state_nominal: + /* New since dumping started; ignore. */ + break; + case prof_tctx_state_dumping: + case prof_tctx_state_purgatory: + prof_tctx_merge_gctx(tctx, tctx->gctx); + break; + default: + not_reached(); } return (NULL); } +/* gctx->lock is held. */ +static prof_tctx_t * +prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) +{ + bool propagate_err = *(bool *)arg; + + if (prof_dump_printf(propagate_err, + " t%"PRIu64": %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]\n", + tctx->tdata->thr_uid, tctx->dump_cnts.curobjs, + tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs, + tctx->dump_cnts.accumbytes)) + return (tctx); + return (NULL); +} + +/* tctx->gctx is held. */ +static prof_tctx_t * +prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) +{ + prof_tctx_t *ret; + + switch (tctx->state) { + case prof_tctx_state_nominal: + /* New since dumping started; ignore. */ + break; + case prof_tctx_state_dumping: + tctx->state = prof_tctx_state_nominal; + break; + case prof_tctx_state_purgatory: + ret = tctx_tree_next(tctxs, tctx); + tctx_tree_remove(tctxs, tctx); + idalloc(tctx); + goto label_return; + default: + not_reached(); + } + + ret = NULL; +label_return: + return (ret); +} + static void -prof_dump_ctx_prep(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx, - prof_ctx_tree_t *ctxs) +prof_dump_gctx_prep(prof_gctx_t *gctx, prof_gctx_tree_t *gctxs) { cassert(config_prof); - malloc_mutex_lock(ctx->lock); + malloc_mutex_lock(gctx->lock); /* - * Increment nlimbo so that ctx won't go away before dump. - * Additionally, link ctx into the dump list so that it is included in + * Increment nlimbo so that gctx won't go away before dump. + * Additionally, link gctx into the dump list so that it is included in * prof_dump()'s second pass. */ - ctx->nlimbo++; - ctx_tree_insert(ctxs, ctx); + gctx->nlimbo++; + gctx_tree_insert(gctxs, gctx); - memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); - thr_cnt_tree_iter(&ctx->thr_cnts, NULL, ctx_sum_iter, (void *)ctx); + memset(&gctx->cnt_summed, 0, sizeof(prof_cnt_t)); - if (ctx->cnt_summed.curobjs != 0) - (*leak_nctx)++; + malloc_mutex_unlock(gctx->lock); +} - /* Add to cnt_all. */ - cnt_all->curobjs += ctx->cnt_summed.curobjs; - cnt_all->curbytes += ctx->cnt_summed.curbytes; - if (opt_prof_accum) { - cnt_all->accumobjs += ctx->cnt_summed.accumobjs; - cnt_all->accumbytes += ctx->cnt_summed.accumbytes; - } +static prof_gctx_t * +prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) +{ + size_t *leak_ngctx = (size_t *)arg; - malloc_mutex_unlock(ctx->lock); + malloc_mutex_lock(gctx->lock); + tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_merge_iter, NULL); + if (gctx->cnt_summed.curobjs != 0) + (*leak_ngctx)++; + malloc_mutex_unlock(gctx->lock); + + return (NULL); +} + +static prof_gctx_t * +prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) +{ + prof_tctx_t *next; + bool destroy_gctx; + + malloc_mutex_lock(gctx->lock); + next = NULL; + do { + next = tctx_tree_iter(&gctx->tctxs, next, prof_tctx_finish_iter, + NULL); + } while (next != NULL); + gctx->nlimbo--; + destroy_gctx = prof_gctx_should_destroy(gctx); + malloc_mutex_unlock(gctx->lock); + if (destroy_gctx) + prof_gctx_maybe_destroy(gctx); + + return (NULL); +} + +static prof_tdata_t * +prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) +{ + prof_cnt_t *cnt_all = (prof_cnt_t *)arg; + + malloc_mutex_lock(tdata->lock); + if (tdata->state != prof_tdata_state_expired) { + size_t tabind; + union { + prof_tctx_t *p; + void *v; + } tctx; + + tdata->dumping = true; + memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); + for (tabind = 0; ckh_iter(&tdata->bt2tctx, &tabind, NULL, + &tctx.v) == false;) + prof_tctx_merge_tdata(tctx.p, tdata); + + cnt_all->curobjs += tdata->cnt_summed.curobjs; + cnt_all->curbytes += tdata->cnt_summed.curbytes; + if (opt_prof_accum) { + cnt_all->accumobjs += tdata->cnt_summed.accumobjs; + cnt_all->accumbytes += tdata->cnt_summed.accumbytes; + } + } else + tdata->dumping = false; + malloc_mutex_unlock(tdata->lock); + + return (NULL); +} + +static prof_tdata_t * +prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) +{ + bool propagate_err = *(bool *)arg; + + if (tdata->dumping == false) + return (NULL); + + if (prof_dump_printf(propagate_err, + " t%"PRIu64": %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]%s%s\n", + tdata->thr_uid, tdata->cnt_summed.curobjs, + tdata->cnt_summed.curbytes, tdata->cnt_summed.accumobjs, + tdata->cnt_summed.accumbytes, + (tdata->thread_name != NULL) ? " " : "", + (tdata->thread_name != NULL) ? tdata->thread_name : "")) + return (tdata); + return (NULL); } static bool prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) { + bool ret; - if (opt_lg_prof_sample == 0) { - if (prof_dump_printf(propagate_err, - "heap profile: %"PRId64": %"PRId64 - " [%"PRIu64": %"PRIu64"] @ heapprofile\n", - cnt_all->curobjs, cnt_all->curbytes, - cnt_all->accumobjs, cnt_all->accumbytes)) - return (true); - } else { - if (prof_dump_printf(propagate_err, - "heap profile: %"PRId64": %"PRId64 - " [%"PRIu64": %"PRIu64"] @ heap_v2/%"PRIu64"\n", - cnt_all->curobjs, cnt_all->curbytes, - cnt_all->accumobjs, cnt_all->accumbytes, - ((uint64_t)1U << opt_lg_prof_sample))) - return (true); - } + if (prof_dump_printf(propagate_err, + "heap_v2/%"PRIu64"\n" + " t*: %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]\n", + ((uint64_t)1U << lg_prof_sample), cnt_all->curobjs, + cnt_all->curbytes, cnt_all->accumobjs, cnt_all->accumbytes)) + return (true); - return (false); -} - -static void -prof_dump_ctx_cleanup_locked(prof_ctx_t *ctx, prof_ctx_tree_t *ctxs) -{ - - ctx->nlimbo--; -} - -static void -prof_dump_ctx_cleanup(prof_ctx_t *ctx, prof_ctx_tree_t *ctxs) -{ - - malloc_mutex_lock(ctx->lock); - prof_dump_ctx_cleanup_locked(ctx, ctxs); - malloc_mutex_unlock(ctx->lock); + malloc_mutex_lock(&tdatas_mtx); + ret = (tdata_tree_iter(&tdatas, NULL, prof_tdata_dump_iter, + (void *)&propagate_err) != NULL); + malloc_mutex_unlock(&tdatas_mtx); + return (ret); } +/* gctx->lock is held. */ static bool -prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, const prof_bt_t *bt, - prof_ctx_tree_t *ctxs) +prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt, + prof_gctx_tree_t *gctxs) { bool ret; unsigned i; cassert(config_prof); - /* - * Current statistics can sum to 0 as a result of unmerged per thread - * statistics. Additionally, interval- and growth-triggered dumps can - * occur between the time a ctx is created and when its statistics are - * filled in. Avoid dumping any ctx that is an artifact of either - * implementation detail. - */ - malloc_mutex_lock(ctx->lock); - if ((opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) || - (opt_prof_accum && ctx->cnt_summed.accumobjs == 0)) { - assert(ctx->cnt_summed.curobjs == 0); - assert(ctx->cnt_summed.curbytes == 0); - assert(ctx->cnt_summed.accumobjs == 0); - assert(ctx->cnt_summed.accumbytes == 0); + /* Avoid dumping such gctx's that have no useful data. */ + if ((opt_prof_accum == false && gctx->cnt_summed.curobjs == 0) || + (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { + assert(gctx->cnt_summed.curobjs == 0); + assert(gctx->cnt_summed.curbytes == 0); + assert(gctx->cnt_summed.accumobjs == 0); + assert(gctx->cnt_summed.accumbytes == 0); ret = false; goto label_return; } - if (prof_dump_printf(propagate_err, "%"PRId64": %"PRId64 - " [%"PRIu64": %"PRIu64"] @", - ctx->cnt_summed.curobjs, ctx->cnt_summed.curbytes, - ctx->cnt_summed.accumobjs, ctx->cnt_summed.accumbytes)) { + if (prof_dump_printf(propagate_err, "@")) { ret = true; goto label_return; } - for (i = 0; i < bt->len; i++) { if (prof_dump_printf(propagate_err, " %#"PRIxPTR, (uintptr_t)bt->vec[i])) { @@ -918,15 +1140,23 @@ prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, const prof_bt_t *bt, } } - if (prof_dump_write(propagate_err, "\n")) { + if (prof_dump_printf(propagate_err, + "\n" + " t*: %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]\n", + gctx->cnt_summed.curobjs, gctx->cnt_summed.curbytes, + gctx->cnt_summed.accumobjs, gctx->cnt_summed.accumbytes)) { + ret = true; + goto label_return; + } + + if (tctx_tree_iter(&gctx->tctxs, NULL, prof_tctx_dump_iter, + (void *)&propagate_err) != NULL) { ret = true; goto label_return; } ret = false; label_return: - prof_dump_ctx_cleanup_locked(ctx, ctxs); - malloc_mutex_unlock(ctx->lock); return (ret); } @@ -980,72 +1210,85 @@ label_return: } static void -prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_nctx, +prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_ngctx, const char *filename) { if (cnt_all->curbytes != 0) { - malloc_printf(": Leak summary: %"PRId64" byte%s, %" - PRId64" object%s, %zu context%s\n", + malloc_printf(": Leak summary: %"PRIu64" byte%s, %" + PRIu64" object%s, %zu context%s\n", cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "", cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "", - leak_nctx, (leak_nctx != 1) ? "s" : ""); + leak_ngctx, (leak_ngctx != 1) ? "s" : ""); malloc_printf( ": Run pprof on \"%s\" for leak detail\n", filename); } } -static prof_ctx_t * -prof_ctx_dump_iter(prof_ctx_tree_t *ctxs, prof_ctx_t *ctx, void *arg) +static prof_gctx_t * +prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) { + prof_gctx_t *ret; bool propagate_err = *(bool *)arg; - if (prof_dump_ctx(propagate_err, ctx, &ctx->bt, ctxs)) - return (ctx_tree_next(ctxs, ctx)); + malloc_mutex_lock(gctx->lock); - return (NULL); -} + if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) { + ret = gctx_tree_next(gctxs, gctx); + goto label_return; + } -static prof_ctx_t * -prof_ctx_cleanup_iter(prof_ctx_tree_t *ctxs, prof_ctx_t *ctx, void *arg) -{ - - prof_dump_ctx_cleanup(ctx, ctxs); - - return (NULL); + ret = NULL; +label_return: + malloc_mutex_unlock(gctx->lock); + return (ret); } static bool prof_dump(bool propagate_err, const char *filename, bool leakcheck) { - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; prof_cnt_t cnt_all; size_t tabind; union { - prof_ctx_t *p; + prof_gctx_t *p; void *v; - } ctx; - size_t leak_nctx; - prof_ctx_tree_t ctxs; - prof_ctx_t *cleanup_start = NULL; + } gctx; + size_t leak_ngctx; + prof_gctx_tree_t gctxs; cassert(config_prof); - prof_tdata = prof_tdata_get(false); - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(false); + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (true); malloc_mutex_lock(&prof_dump_mtx); + prof_enter(tdata); - /* Merge per thread profile stats, and sum them in cnt_all. */ + /* + * Put gctx's in limbo and clear their counters in preparation for + * summing. + */ + gctx_tree_new(&gctxs); + for (tabind = 0; ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v) == false;) + prof_dump_gctx_prep(gctx.p, &gctxs); + + /* + * Iterate over tdatas, and for the non-expired ones snapshot their tctx + * stats and merge them into the associated gctx's. + */ memset(&cnt_all, 0, sizeof(prof_cnt_t)); - leak_nctx = 0; - ctx_tree_new(&ctxs); - prof_enter(prof_tdata); - for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) - prof_dump_ctx_prep(ctx.p, &cnt_all, &leak_nctx, &ctxs); - prof_leave(prof_tdata); + malloc_mutex_lock(&tdatas_mtx); + tdata_tree_iter(&tdatas, NULL, prof_tdata_merge_iter, (void *)&cnt_all); + malloc_mutex_unlock(&tdatas_mtx); + + /* Merge tctx stats into gctx's. */ + leak_ngctx = 0; + gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx); + + prof_leave(tdata); /* Create dump file. */ if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) @@ -1055,10 +1298,9 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) if (prof_dump_header(propagate_err, &cnt_all)) goto label_write_error; - /* Dump per ctx profile stats. */ - cleanup_start = ctx_tree_iter(&ctxs, NULL, prof_ctx_dump_iter, - (void *)&propagate_err); - if (cleanup_start != NULL) + /* Dump per gctx profile stats. */ + if (gctx_tree_iter(&gctxs, NULL, prof_gctx_dump_iter, + (void *)&propagate_err) != NULL) goto label_write_error; /* Dump /proc//maps if possible. */ @@ -1068,19 +1310,17 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) if (prof_dump_close(propagate_err)) goto label_open_close_error; + gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, NULL); malloc_mutex_unlock(&prof_dump_mtx); if (leakcheck) - prof_leakcheck(&cnt_all, leak_nctx, filename); + prof_leakcheck(&cnt_all, leak_ngctx, filename); return (false); label_write_error: prof_dump_close(propagate_err); label_open_close_error: - if (cleanup_start != NULL) { - ctx_tree_iter(&ctxs, cleanup_start, prof_ctx_cleanup_iter, - NULL); - } + gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, NULL); malloc_mutex_unlock(&prof_dump_mtx); return (true); } @@ -1128,18 +1368,18 @@ prof_fdump(void) void prof_idump(void) { - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; char filename[PATH_MAX + 1]; cassert(config_prof); if (prof_booted == false) return; - prof_tdata = prof_tdata_get(false); - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(false); + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return; - if (prof_tdata->enq) { - prof_tdata->enq_idump = true; + if (tdata->enq) { + tdata->enq_idump = true; return; } @@ -1178,18 +1418,18 @@ prof_mdump(const char *filename) void prof_gdump(void) { - prof_tdata_t *prof_tdata; + prof_tdata_t *tdata; char filename[DUMP_FILENAME_BUFSIZE]; cassert(config_prof); if (prof_booted == false) return; - prof_tdata = prof_tdata_get(false); - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(false); + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return; - if (prof_tdata->enq) { - prof_tdata->enq_gdump = true; + if (tdata->enq) { + tdata->enq_gdump = true; return; } @@ -1225,81 +1465,233 @@ prof_bt_keycomp(const void *k1, const void *k2) return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); } -prof_tdata_t * -prof_tdata_init(void) +JEMALLOC_INLINE_C uint64_t +prof_thr_uid_alloc(void) { - prof_tdata_t *prof_tdata; + + return (atomic_add_uint64(&next_thr_uid, 1) - 1); +} + +static prof_tdata_t * +prof_tdata_init_impl(uint64_t thr_uid) +{ + prof_tdata_t *tdata; cassert(config_prof); /* Initialize an empty cache for this thread. */ - prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); - if (prof_tdata == NULL) + tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); + if (tdata == NULL) return (NULL); - if (ckh_new(&prof_tdata->bt2cnt, PROF_CKH_MINITEMS, + tdata->lock = prof_tdata_mutex_choose(thr_uid); + tdata->thr_uid = thr_uid; + tdata->thread_name = NULL; + tdata->state = prof_tdata_state_attached; + + if (ckh_new(&tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) { - idalloc(prof_tdata); + idalloc(tdata); return (NULL); } - prof_tdata->prng_state = (uint64_t)(uintptr_t)prof_tdata; - prof_sample_threshold_update(prof_tdata); + tdata->prng_state = (uint64_t)(uintptr_t)tdata; + prof_sample_threshold_update(tdata); - prof_tdata->enq = false; - prof_tdata->enq_idump = false; - prof_tdata->enq_gdump = false; + tdata->enq = false; + tdata->enq_idump = false; + tdata->enq_gdump = false; - prof_tdata_tsd_set(&prof_tdata); + tdata->dumping = false; + tdata->active = true; - return (prof_tdata); + prof_tdata_tsd_set(&tdata); + + malloc_mutex_lock(&tdatas_mtx); + tdata_tree_insert(&tdatas, tdata); + malloc_mutex_unlock(&tdatas_mtx); + + return (tdata); +} + +prof_tdata_t * +prof_tdata_init(void) +{ + + return (prof_tdata_init_impl(prof_thr_uid_alloc())); +} + +prof_tdata_t * +prof_tdata_reinit(prof_tdata_t *tdata) +{ + + return (prof_tdata_init_impl(tdata->thr_uid)); +} + +/* tdata->lock must be held. */ +static bool +prof_tdata_should_destroy(prof_tdata_t *tdata) +{ + + if (tdata->state == prof_tdata_state_attached) + return (false); + if (ckh_count(&tdata->bt2tctx) != 0) + return (false); + return (true); +} + +static void +prof_tdata_destroy(prof_tdata_t *tdata) +{ + + assert(prof_tdata_should_destroy(tdata)); + + malloc_mutex_lock(&tdatas_mtx); + tdata_tree_remove(&tdatas, tdata); + malloc_mutex_unlock(&tdatas_mtx); + + if (tdata->thread_name != NULL) + idalloc(tdata->thread_name); + ckh_delete(&tdata->bt2tctx); + idalloc(tdata); +} + +static void +prof_tdata_state_transition(prof_tdata_t *tdata, prof_tdata_state_t state) +{ + bool destroy_tdata; + + malloc_mutex_lock(tdata->lock); + if (tdata->state != state) { + tdata->state = state; + destroy_tdata = prof_tdata_should_destroy(tdata); + } else + destroy_tdata = false; + malloc_mutex_unlock(tdata->lock); + if (destroy_tdata) + prof_tdata_destroy(tdata); +} + +static void +prof_tdata_detach(prof_tdata_t *tdata) +{ + + prof_tdata_state_transition(tdata, prof_tdata_state_detached); +} + +static void +prof_tdata_expire(prof_tdata_t *tdata) +{ + + prof_tdata_state_transition(tdata, prof_tdata_state_expired); +} + +static prof_tdata_t * +prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) +{ + + prof_tdata_expire(tdata); + return (NULL); +} + +void +prof_reset(size_t lg_sample) +{ + + assert(lg_sample < (sizeof(uint64_t) << 3)); + + malloc_mutex_lock(&prof_dump_mtx); + malloc_mutex_lock(&tdatas_mtx); + + lg_prof_sample = lg_sample; + tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, NULL); + + malloc_mutex_unlock(&tdatas_mtx); + malloc_mutex_unlock(&prof_dump_mtx); } void prof_tdata_cleanup(void *arg) { - prof_tdata_t *prof_tdata = *(prof_tdata_t **)arg; + prof_tdata_t *tdata = *(prof_tdata_t **)arg; cassert(config_prof); - if (prof_tdata == PROF_TDATA_STATE_REINCARNATED) { + if (tdata == PROF_TDATA_STATE_REINCARNATED) { /* * Another destructor deallocated memory after this destructor - * was called. Reset prof_tdata to PROF_TDATA_STATE_PURGATORY - * in order to receive another callback. + * was called. Reset tdata to PROF_TDATA_STATE_PURGATORY in + * order to receive another callback. */ - prof_tdata = PROF_TDATA_STATE_PURGATORY; - prof_tdata_tsd_set(&prof_tdata); - } else if (prof_tdata == PROF_TDATA_STATE_PURGATORY) { + tdata = PROF_TDATA_STATE_PURGATORY; + prof_tdata_tsd_set(&tdata); + } else if (tdata == PROF_TDATA_STATE_PURGATORY) { /* * The previous time this destructor was called, we set the key * to PROF_TDATA_STATE_PURGATORY so that other destructors - * wouldn't cause re-creation of the prof_tdata. This time, do + * wouldn't cause re-creation of the tdata. This time, do * nothing, so that the destructor will not be called again. */ - } else if (prof_tdata != NULL) { - union { - prof_thr_cnt_t *p; - void *v; - } cnt; - size_t tabind; - - /* - * Iteratively merge cnt's into the global stats and delete - * them. - */ - for (tabind = 0; ckh_iter(&prof_tdata->bt2cnt, &tabind, NULL, - &cnt.v);) { - prof_ctx_merge(cnt.p->ctx, cnt.p); - idalloc(cnt.v); - } - ckh_delete(&prof_tdata->bt2cnt); - idalloc(prof_tdata); - prof_tdata = PROF_TDATA_STATE_PURGATORY; - prof_tdata_tsd_set(&prof_tdata); + } else if (tdata != NULL) { + prof_tdata_detach(tdata); + tdata = PROF_TDATA_STATE_PURGATORY; + prof_tdata_tsd_set(&tdata); } } +const char * +prof_thread_name_get(void) +{ + prof_tdata_t *tdata = prof_tdata_get(true); + if (tdata == NULL) + return (NULL); + return (tdata->thread_name); +} + +bool +prof_thread_name_set(const char *thread_name) +{ + prof_tdata_t *tdata; + size_t size; + char *s; + + tdata = prof_tdata_get(true); + if (tdata == NULL) + return (true); + + size = strlen(thread_name) + 1; + s = imalloc(size); + if (s == NULL) + return (true); + + memcpy(s, thread_name, size); + if (tdata->thread_name != NULL) + idalloc(tdata->thread_name); + tdata->thread_name = s; + return (false); +} + +bool +prof_thread_active_get(void) +{ + prof_tdata_t *tdata = prof_tdata_get(true); + if (tdata == NULL) + return (false); + return (tdata->active); +} + +bool +prof_thread_active_set(bool active) +{ + prof_tdata_t *tdata; + + tdata = prof_tdata_get(true); + if (tdata == NULL) + return (true); + tdata->active = active; + return (false); +} + void prof_boot0(void) { @@ -1345,10 +1737,12 @@ prof_boot2(void) if (opt_prof) { unsigned i; - if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash, + lg_prof_sample = opt_lg_prof_sample; + + if (ckh_new(&bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) return (true); - if (malloc_mutex_init(&bt2ctx_mtx)) + if (malloc_mutex_init(&bt2gctx_mtx)) return (true); if (prof_tdata_tsd_boot()) { malloc_write( @@ -1356,6 +1750,12 @@ prof_boot2(void) abort(); } + tdata_tree_new(&tdatas); + if (malloc_mutex_init(&tdatas_mtx)) + return (true); + + next_thr_uid = 0; + if (malloc_mutex_init(&prof_dump_seq_mtx)) return (true); if (malloc_mutex_init(&prof_dump_mtx)) @@ -1367,12 +1767,21 @@ prof_boot2(void) abort(); } - ctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS * + gctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS * sizeof(malloc_mutex_t)); - if (ctx_locks == NULL) + if (gctx_locks == NULL) return (true); for (i = 0; i < PROF_NCTX_LOCKS; i++) { - if (malloc_mutex_init(&ctx_locks[i])) + if (malloc_mutex_init(&gctx_locks[i])) + return (true); + } + + tdata_locks = (malloc_mutex_t *)base_alloc(PROF_NTDATA_LOCKS * + sizeof(malloc_mutex_t)); + if (tdata_locks == NULL) + return (true); + for (i = 0; i < PROF_NTDATA_LOCKS; i++) { + if (malloc_mutex_init(&tdata_locks[i])) return (true); } } @@ -1397,10 +1806,10 @@ prof_prefork(void) if (opt_prof) { unsigned i; - malloc_mutex_prefork(&bt2ctx_mtx); + malloc_mutex_prefork(&bt2gctx_mtx); malloc_mutex_prefork(&prof_dump_seq_mtx); for (i = 0; i < PROF_NCTX_LOCKS; i++) - malloc_mutex_prefork(&ctx_locks[i]); + malloc_mutex_prefork(&gctx_locks[i]); } } @@ -1412,9 +1821,9 @@ prof_postfork_parent(void) unsigned i; for (i = 0; i < PROF_NCTX_LOCKS; i++) - malloc_mutex_postfork_parent(&ctx_locks[i]); + malloc_mutex_postfork_parent(&gctx_locks[i]); malloc_mutex_postfork_parent(&prof_dump_seq_mtx); - malloc_mutex_postfork_parent(&bt2ctx_mtx); + malloc_mutex_postfork_parent(&bt2gctx_mtx); } } @@ -1426,9 +1835,9 @@ prof_postfork_child(void) unsigned i; for (i = 0; i < PROF_NCTX_LOCKS; i++) - malloc_mutex_postfork_child(&ctx_locks[i]); + malloc_mutex_postfork_child(&gctx_locks[i]); malloc_mutex_postfork_child(&prof_dump_seq_mtx); - malloc_mutex_postfork_child(&bt2ctx_mtx); + malloc_mutex_postfork_child(&bt2gctx_mtx); } } diff --git a/src/stats.c b/src/stats.c index a0eb2971..db34275e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -441,7 +441,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, } if ((err = je_mallctl("opt.prof", &bv, &bsz, NULL, 0)) == 0 && bv) { - CTL_GET("opt.lg_prof_sample", &sv, size_t); + CTL_GET("prof.lg_sample", &sv, size_t); malloc_cprintf(write_cb, cbopaque, "Average profile sample interval: %"PRIu64 " (2^%zu)\n", (((uint64_t)1U) << sv), sv); From 3e24afa28e01b743a9f7fa1d42acb67e079d8187 Mon Sep 17 00:00:00 2001 From: Sara Golemon Date: Mon, 18 Aug 2014 13:06:39 -0700 Subject: [PATCH 081/352] Test for availability of malloc hooks via autoconf __*_hook() is glibc, but on at least one glibc platform (homebrew), the __GLIBC__ define isn't set correctly and we miss being able to use these hooks. Do a feature test for it during configuration so that we enable it anywhere the hooks are actually available. --- configure.ac | 31 +++++++++++++++++++ .../internal/jemalloc_internal_defs.h.in | 6 ++++ src/jemalloc.c | 4 ++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index ede5f70f..6f8fd3fd 100644 --- a/configure.ac +++ b/configure.ac @@ -1340,6 +1340,37 @@ if test "x${enable_zone_allocator}" = "x1" ; then AC_DEFINE_UNQUOTED(JEMALLOC_ZONE_VERSION, [$JEMALLOC_ZONE_VERSION]) fi +dnl ============================================================================ +dnl Check for glibc malloc hooks + +JE_COMPILABLE([glibc malloc hook], [ +#include + +extern void (* __free_hook)(void *ptr); +extern void *(* __malloc_hook)(size_t size); +extern void *(* __realloc_hook)(void *ptr, size_t size); +], [ + void *ptr = 0L; + if (__malloc_hook) ptr = __malloc_hook(1); + if (__realloc_hook) ptr = __realloc_hook(ptr, 2); + if (__free_hook && ptr) __free_hook(ptr); +], [je_cv_glibc_malloc_hook]) +if test "x${je_cv_glibc_malloc_hook}" = "xyes" ; then + AC_DEFINE([JEMALLOC_GLIBC_MALLOC_HOOK], [ ]) +fi + +JE_COMPILABLE([glibc memalign hook], [ +#include + +extern void *(* __memalign_hook)(size_t alignment, size_t size); +], [ + void *ptr = 0L; + if (__memalign_hook) ptr = __memalign_hook(16, 7); +], [je_cv_glibc_memalign_hook]) +if test "x${je_cv_glibc_memalign_hook}" = "xyes" ; then + AC_DEFINE([JEMALLOC_GLIBC_MEMALIGN_HOOK], [ ]) +fi + dnl ============================================================================ dnl Check for typedefs, structures, and compiler characteristics. AC_HEADER_STDBOOL diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 93716b0a..955582ee 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -209,4 +209,10 @@ /* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ #undef LG_SIZEOF_INTMAX_T +/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook) */ +#undef JEMALLOC_GLIBC_MALLOC_HOOK + +/* glibc memalign hook */ +#undef JEMALLOC_GLIBC_MEMALIGN_HOOK + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/src/jemalloc.c b/src/jemalloc.c index 2d01272e..9df70018 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1336,7 +1336,7 @@ je_valloc(size_t size) #define is_malloc_(a) malloc_is_ ## a #define is_malloc(a) is_malloc_(a) -#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) +#if ((is_malloc(je_malloc) == 1) && defined(JEMALLOC_GLIBC_MALLOC_HOOK)) /* * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible * to inconsistently reference libc's malloc(3)-compatible functions @@ -1349,8 +1349,10 @@ je_valloc(size_t size) JEMALLOC_EXPORT void (*__free_hook)(void *ptr) = je_free; JEMALLOC_EXPORT void *(*__malloc_hook)(size_t size) = je_malloc; JEMALLOC_EXPORT void *(*__realloc_hook)(void *ptr, size_t size) = je_realloc; +# ifdef JEMALLOC_GLIBC_MEMALIGN_HOOK JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = je_memalign; +# endif #endif /* From 58799f6d1c1f58053f4aac1b100ce9049c868039 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Tue, 26 Aug 2014 21:28:31 -0700 Subject: [PATCH 082/352] Remove junk filling in tcache_bin_flush_small(). Junk filling is done in arena_dalloc_bin_locked(), so arena_alloc_junk_small() is redundant. Also, we should use arena_dalloc_junk_small() instead of arena_alloc_junk_small(). --- src/tcache.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/tcache.c b/src/tcache.c index 868f2d77..4fbc94cc 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -120,10 +120,6 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); - if (config_fill && opt_junk) { - arena_alloc_junk_small(ptr, - &arena_bin_info[binind], true); - } arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); } else { From a5a658ab48f7dfa7fd134e505ef23304eaa0ce54 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 2 Sep 2014 15:07:07 -0700 Subject: [PATCH 083/352] Make VERSION generation more robust. Relax the "are we in a git repo?" check to succeed even if the top level jemalloc directory is not at the top level of the git repo. Add git tag filtering so that only version triplets match when generating VERSION. Add fallback bogus VERSION creation, so that in the worst case, rather than generating empty values for e.g. JEMALLOC_VERSION_MAJOR, configuration ends up generating useless constants. --- configure.ac | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 6f8fd3fd..3b658852 100644 --- a/configure.ac +++ b/configure.ac @@ -1029,11 +1029,33 @@ dnl ============================================================================ dnl jemalloc configuration. dnl -dnl Set VERSION if source directory has an embedded git repository or is a git submodule. -if test -e "${srcroot}.git" ; then - git describe --long --abbrev=40 > ${srcroot}VERSION +dnl Set VERSION if source directory is inside a git repository. +if test "x`git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then + dnl Pattern globs aren't powerful enough to match both single- and + dnl double-digit version numbers, so iterate over patterns to support up to + dnl version 99.99.99 without any accidental matches. + rm -f "${srcroot}VERSION" + for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ + '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do + if test ! -e "${srcroot}VERSION" ; then + git describe --long --abbrev=40 --match="${pattern}" > "${srcroot}VERSION.tmp" 2>/dev/null + if test $? -eq 0 ; then + mv "${srcroot}VERSION.tmp" "${srcroot}VERSION" + break + fi + fi + done fi -jemalloc_version=`cat ${srcroot}VERSION` +rm -f "${srcroot}VERSION.tmp" +if test ! -e "${srcroot}VERSION" ; then + AC_MSG_RESULT( + [Missing VERSION file, and unable to generate it; creating bogus VERSION]) + echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${srcroot}VERSION" +fi +jemalloc_version=`cat "${srcroot}VERSION"` jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` From f34f6037e8d9836f7cddc02ad349dc72964bbcc7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 2 Sep 2014 17:49:29 -0700 Subject: [PATCH 084/352] Disable autom4te cache. --- .autom4te.cfg | 3 +++ .gitignore | 2 -- Makefile.in | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 .autom4te.cfg diff --git a/.autom4te.cfg b/.autom4te.cfg new file mode 100644 index 00000000..fe2424db --- /dev/null +++ b/.autom4te.cfg @@ -0,0 +1,3 @@ +begin-language: "Autoconf-without-aclocal-m4" +args: --no-cache +end-language: "Autoconf-without-aclocal-m4" diff --git a/.gitignore b/.gitignore index 4c408ec2..ec9c0b92 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ /*.gcov.* -/autom4te.cache/ - /bin/jemalloc.sh /config.stamp diff --git a/Makefile.in b/Makefile.in index dfafe455..b5f0ee90 100644 --- a/Makefile.in +++ b/Makefile.in @@ -400,7 +400,6 @@ clean: rm -f $(objroot)*.gcov.* distclean: clean - rm -rf $(objroot)autom4te.cache rm -f $(objroot)bin/jemalloc.sh rm -f $(objroot)config.log rm -f $(objroot)config.status From ff6a31d3b92b7c63446ce645341d2bbd77b67dc6 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Fri, 29 Aug 2014 13:34:40 -0700 Subject: [PATCH 085/352] Refactor chunk map. Break the chunk map into two separate arrays, in order to improve cache locality. This is related to issue #23. --- include/jemalloc/internal/arena.h | 108 +++++---- include/jemalloc/internal/chunk.h | 1 + include/jemalloc/internal/private_symbols.txt | 4 +- include/jemalloc/internal/size_classes.sh | 2 +- src/arena.c | 208 +++++++++--------- src/chunk.c | 1 + src/tcache.c | 11 +- 7 files changed, 186 insertions(+), 149 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index f3f6426c..986bea92 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -43,7 +43,8 @@ */ #define LG_DIRTY_MULT_DEFAULT 3 -typedef struct arena_chunk_map_s arena_chunk_map_t; +typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; +typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; typedef struct arena_chunk_s arena_chunk_t; typedef struct arena_run_s arena_run_t; typedef struct arena_bin_info_s arena_bin_info_t; @@ -55,34 +56,7 @@ typedef struct arena_s arena_t; #ifdef JEMALLOC_H_STRUCTS /* Each element of the chunk map corresponds to one page within the chunk. */ -struct arena_chunk_map_s { -#ifndef JEMALLOC_PROF - /* - * Overlay prof_tctx in order to allow it to be referenced by dead code. - * Such antics aren't warranted for per arena data structures, but - * chunk map overhead accounts for a percentage of memory, rather than - * being just a fixed cost. - */ - union { -#endif - /* - * Linkage for run trees. There are two disjoint uses: - * - * 1) arena_t's runs_avail tree. - * 2) arena_run_t conceptually uses this linkage for in-use non-full - * runs, rather than directly embedding linkage. - */ - rb_node(arena_chunk_map_t) rb_link; - - /* Profile counters, used for large object runs. */ - prof_tctx_t *prof_tctx; -#ifndef JEMALLOC_PROF - }; /* union { ... }; */ -#endif - - /* Linkage for list of dirty runs. */ - ql_elm(arena_chunk_map_t) dr_link; - +struct arena_chunk_map_bits_s { /* * Run address (or size) and various flags are stored together. The bit * layout looks like (assuming 32-bit system): @@ -149,9 +123,43 @@ struct arena_chunk_map_s { #define CHUNK_MAP_ALLOCATED ((size_t)0x1U) #define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED }; -typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; -typedef rb_tree(arena_chunk_map_t) arena_run_tree_t; -typedef ql_head(arena_chunk_map_t) arena_chunk_mapelms_t; + +/* + * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just + * like arena_chunk_map_bits_t. Two separate arrays are stored within each + * chunk header in order to improve cache locality. + */ +struct arena_chunk_map_misc_s { +#ifndef JEMALLOC_PROF + /* + * Overlay prof_tctx in order to allow it to be referenced by dead code. + * Such antics aren't warranted for per arena data structures, but + * chunk map overhead accounts for a percentage of memory, rather than + * being just a fixed cost. + */ + union { +#endif + /* + * Linkage for run trees. There are two disjoint uses: + * + * 1) arena_t's runs_avail tree. + * 2) arena_run_t conceptually uses this linkage for in-use non-full + * runs, rather than directly embedding linkage. + */ + rb_node(arena_chunk_map_misc_t) rb_link; + + /* Profile counters, used for large object runs. */ + prof_tctx_t *prof_tctx; +#ifndef JEMALLOC_PROF + }; /* union { ... }; */ +#endif + + /* Linkage for list of dirty runs. */ + ql_elm(arena_chunk_map_misc_t) dr_link; +}; +typedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t; +typedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; +typedef ql_head(arena_chunk_map_misc_t) arena_chunk_miscelms_t; /* Arena chunk header. */ struct arena_chunk_s { @@ -164,7 +172,7 @@ struct arena_chunk_s { * need to be tracked in the map. This omission saves a header page * for common chunk sizes (e.g. 4 MiB). */ - arena_chunk_map_t map[1]; /* Dynamically sized. */ + arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ }; struct arena_run_s { @@ -335,7 +343,7 @@ struct arena_s { arena_avail_tree_t runs_avail; /* List of dirty runs this arena manages. */ - arena_chunk_mapelms_t runs_dirty; + arena_chunk_miscelms_t runs_dirty; /* * user-configureable chunk allocation and deallocation functions. @@ -393,9 +401,9 @@ void *arena_malloc_large(arena_t *arena, size_t size, bool zero); void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); void arena_prof_promoted(const void *ptr, size_t size); void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, - arena_chunk_map_t *mapelm); + arena_chunk_map_bits_t *bitselm); void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, - size_t pageind, arena_chunk_map_t *mapelm); + size_t pageind, arena_chunk_map_bits_t *bitselm); void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind); #ifdef JEMALLOC_JET @@ -439,7 +447,10 @@ size_t small_bin2size(size_t binind); size_t small_s2u_compute(size_t size); size_t small_s2u_lookup(size_t size); size_t small_s2u(size_t size); -arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); +arena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, + size_t pageind); +arena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, + size_t pageind); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbitsp_read(size_t *mapbitsp); size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); @@ -623,21 +634,32 @@ small_s2u(size_t size) # endif /* JEMALLOC_ARENA_INLINE_A */ # ifdef JEMALLOC_ARENA_INLINE_B -JEMALLOC_ALWAYS_INLINE arena_chunk_map_t * -arena_mapp_get(arena_chunk_t *chunk, size_t pageind) +JEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * +arena_bitselm_get(arena_chunk_t *chunk, size_t pageind) { assert(pageind >= map_bias); assert(pageind < chunk_npages); - return (&chunk->map[pageind-map_bias]); + return (&chunk->map_bits[pageind-map_bias]); +} + +JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * +arena_miscelm_get(arena_chunk_t *chunk, size_t pageind) +{ + + assert(pageind >= map_bias); + assert(pageind < chunk_npages); + + return ((arena_chunk_map_misc_t *)((uintptr_t)chunk + + (uintptr_t)map_misc_offset) + pageind-map_bias); } JEMALLOC_ALWAYS_INLINE size_t * arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) { - return (&arena_mapp_get(chunk, pageind)->bits); + return (&arena_bitselm_get(chunk, pageind)->bits); } JEMALLOC_ALWAYS_INLINE size_t @@ -1005,7 +1027,7 @@ arena_prof_tctx_get(const void *ptr) if ((mapbits & CHUNK_MAP_LARGE) == 0) ret = (prof_tctx_t *)(uintptr_t)1U; else - ret = arena_mapp_get(chunk, pageind)->prof_tctx; + ret = arena_miscelm_get(chunk, pageind)->prof_tctx; return (ret); } @@ -1025,7 +1047,7 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) assert(arena_mapbits_allocated_get(chunk, pageind) != 0); if (arena_mapbits_large_get(chunk, pageind) != 0) - arena_mapp_get(chunk, pageind)->prof_tctx = tctx; + arena_miscelm_get(chunk, pageind)->prof_tctx = tctx; } JEMALLOC_ALWAYS_INLINE void * diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index f3bfbe08..27aa0adf 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -41,6 +41,7 @@ extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ extern size_t chunk_npages; extern size_t map_bias; /* Number of arena chunk header pages. */ +extern size_t map_misc_offset; extern size_t arena_maxclass; /* Max size class for arenas. */ void *chunk_alloc_base(size_t size); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 13505455..9ca139ab 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -4,6 +4,7 @@ a0malloc arena_alloc_junk_small arena_bin_index arena_bin_info +arena_bitselm_get arena_boot arena_chunk_alloc_huge arena_chunk_dalloc_huge @@ -38,8 +39,8 @@ arena_mapbits_unzeroed_set arena_mapbitsp_get arena_mapbitsp_read arena_mapbitsp_write -arena_mapp_get arena_maxclass +arena_miscelm_get arena_new arena_palloc arena_postfork_child @@ -254,6 +255,7 @@ malloc_vcprintf malloc_vsnprintf malloc_write map_bias +map_misc_offset mb_write mutex_boot narenas_auto diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 3edebf23..379d36c2 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -240,7 +240,7 @@ cat < 255) # error "Too many small size classes" diff --git a/src/arena.c b/src/arena.c index 1263269e..d9dda832 100644 --- a/src/arena.c +++ b/src/arena.c @@ -61,55 +61,57 @@ static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, /******************************************************************************/ JEMALLOC_INLINE_C size_t -arena_mapelm_to_pageind(arena_chunk_map_t *mapelm) +arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm) { - uintptr_t map_offset = - CHUNK_ADDR2OFFSET(mapelm) - offsetof(arena_chunk_t, map); + size_t offset = CHUNK_ADDR2OFFSET(miscelm); - return ((map_offset / sizeof(arena_chunk_map_t)) + map_bias); + return ((offset - map_misc_offset) / sizeof(arena_chunk_map_misc_t) + + map_bias); } JEMALLOC_INLINE_C size_t -arena_mapelm_to_bits(arena_chunk_map_t *mapelm) +arena_miscelm_to_bits(arena_chunk_map_misc_t *miscelm) { + arena_chunk_t *chunk = CHUNK_ADDR2BASE(miscelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); - return (mapelm->bits); + return arena_mapbits_get(chunk, pageind); } static inline int -arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) +arena_run_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) { - uintptr_t a_mapelm = (uintptr_t)a; - uintptr_t b_mapelm = (uintptr_t)b; + uintptr_t a_miscelm = (uintptr_t)a; + uintptr_t b_miscelm = (uintptr_t)b; assert(a != NULL); assert(b != NULL); - return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); + return ((a_miscelm > b_miscelm) - (a_miscelm < b_miscelm)); } /* Generate red-black tree functions. */ -rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, +rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_misc_t, rb_link, arena_run_comp) static inline int -arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) +arena_avail_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) { int ret; size_t a_size; - size_t b_size = arena_mapelm_to_bits(b) & ~PAGE_MASK; - uintptr_t a_mapelm = (uintptr_t)a; - uintptr_t b_mapelm = (uintptr_t)b; + size_t b_size = arena_miscelm_to_bits(b) & ~PAGE_MASK; + uintptr_t a_miscelm = (uintptr_t)a; + uintptr_t b_miscelm = (uintptr_t)b; - if (a_mapelm & CHUNK_MAP_KEY) - a_size = a_mapelm & ~PAGE_MASK; - else - a_size = arena_mapelm_to_bits(a) & ~PAGE_MASK; + if (a_miscelm & CHUNK_MAP_KEY) + a_size = a_miscelm & ~PAGE_MASK; + else + a_size = arena_miscelm_to_bits(a) & ~PAGE_MASK; ret = (a_size > b_size) - (a_size < b_size); if (ret == 0) { - if (!(a_mapelm & CHUNK_MAP_KEY)) - ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); + if (!(a_miscelm & CHUNK_MAP_KEY)) + ret = (a_miscelm > b_miscelm) - (a_miscelm < b_miscelm); else { /* * Treat keys as if they are lower than anything else. @@ -122,8 +124,8 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) } /* Generate red-black tree functions. */ -rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, - rb_link, arena_avail_comp) +rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, + arena_chunk_map_misc_t, rb_link, arena_avail_comp) static void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, @@ -132,7 +134,7 @@ arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); - arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, + arena_avail_tree_insert(&arena->runs_avail, arena_miscelm_get(chunk, pageind)); } @@ -143,7 +145,7 @@ arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); - arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, + arena_avail_tree_remove(&arena->runs_avail, arena_miscelm_get(chunk, pageind)); } @@ -151,14 +153,14 @@ static void arena_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, size_t npages) { - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - ql_elm_new(mapelm, dr_link); - ql_tail_insert(&arena->runs_dirty, mapelm, dr_link); + ql_elm_new(miscelm, dr_link); + ql_tail_insert(&arena->runs_dirty, miscelm, dr_link); arena->ndirty += npages; } @@ -166,13 +168,13 @@ static void arena_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, size_t npages) { - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - ql_remove(&arena->runs_dirty, mapelm, dr_link); + ql_remove(&arena->runs_dirty, miscelm, dr_link); arena->ndirty -= npages; } @@ -532,16 +534,17 @@ arena_chunk_init_hard(arena_t *arena) */ if (zero == false) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( - (void *)arena_mapp_get(chunk, map_bias+1), - (size_t)((uintptr_t) arena_mapp_get(chunk, chunk_npages-1) - - (uintptr_t)arena_mapp_get(chunk, map_bias+1))); + (void *)arena_bitselm_get(chunk, map_bias+1), + (size_t)((uintptr_t) arena_bitselm_get(chunk, + chunk_npages-1) - (uintptr_t)arena_bitselm_get(chunk, + map_bias+1))); for (i = map_bias+1; i < chunk_npages-1; i++) arena_mapbits_unzeroed_set(chunk, i, unzeroed); } else { - JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, - map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, - chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, - map_bias+1))); + JEMALLOC_VALGRIND_MAKE_MEM_DEFINED((void + *)arena_bitselm_get(chunk, map_bias+1), (size_t)((uintptr_t) + arena_bitselm_get(chunk, chunk_npages-1) - + (uintptr_t)arena_bitselm_get(chunk, map_bias+1))); if (config_debug) { for (i = map_bias+1; i < chunk_npages-1; i++) { assert(arena_mapbits_unzeroed_get(chunk, i) == @@ -641,14 +644,14 @@ static arena_run_t * arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { arena_run_t *run; - arena_chunk_map_t *mapelm; - arena_chunk_map_t *key; + arena_chunk_map_misc_t *miscelm; + arena_chunk_map_misc_t *key; - key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); - mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = arena_mapelm_to_pageind(mapelm); + key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); + miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); + if (miscelm != NULL) { + arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(miscelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << LG_PAGE)); @@ -695,14 +698,14 @@ static arena_run_t * arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) { arena_run_t *run; - arena_chunk_map_t *mapelm; - arena_chunk_map_t *key; + arena_chunk_map_misc_t *miscelm; + arena_chunk_map_misc_t *key; - key = (arena_chunk_map_t *)(size | CHUNK_MAP_KEY); - mapelm = arena_avail_tree_nsearch(&arena->runs_avail, key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = arena_mapelm_to_pageind(mapelm); + key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); + miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); + if (miscelm != NULL) { + arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(miscelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << LG_PAGE)); @@ -769,13 +772,13 @@ static size_t arena_dirty_count(arena_t *arena) { size_t ndirty = 0; - arena_chunk_map_t *mapelm; + arena_chunk_map_misc_t *miscelm; arena_chunk_t *chunk; size_t pageind, npages; - ql_foreach(mapelm, &arena->runs_dirty, dr_link) { - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = arena_mapelm_to_pageind(mapelm); + ql_foreach(miscelm, &arena->runs_dirty, dr_link) { + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + pageind = arena_miscelm_to_pageind(miscelm); assert(arena_mapbits_allocated_get(chunk, pageind) == 0); assert(arena_mapbits_large_get(chunk, pageind) == 0); assert(arena_mapbits_dirty_get(chunk, pageind) != 0); @@ -808,16 +811,17 @@ arena_compute_npurge(arena_t *arena, bool all) static size_t arena_stash_dirty(arena_t *arena, bool all, size_t npurge, - arena_chunk_mapelms_t *mapelms) + arena_chunk_miscelms_t *miscelms) { - arena_chunk_map_t *mapelm; + arena_chunk_map_misc_t *miscelm; size_t nstashed = 0; /* Add at least npurge pages to purge_list. */ - for (mapelm = ql_first(&arena->runs_dirty); mapelm != NULL; - mapelm = ql_first(&arena->runs_dirty)) { - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - size_t pageind = arena_mapelm_to_pageind(mapelm); + for (miscelm = ql_first(&arena->runs_dirty); miscelm != NULL; + miscelm = ql_first(&arena->runs_dirty)) { + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); size_t run_size = arena_mapbits_unallocated_size_get(chunk, pageind); size_t npages = run_size >> LG_PAGE; @@ -838,8 +842,8 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, /* Temporarily allocate the free dirty run. */ arena_run_split_large(arena, run, run_size, false); /* Append to purge_list for later processing. */ - ql_elm_new(mapelm, dr_link); - ql_tail_insert(mapelms, mapelm, dr_link); + ql_elm_new(miscelm, dr_link); + ql_tail_insert(miscelms, miscelm, dr_link); nstashed += npages; @@ -851,10 +855,10 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, } static size_t -arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms) +arena_purge_stashed(arena_t *arena, arena_chunk_miscelms_t *miscelms) { size_t npurged, nmadvise; - arena_chunk_map_t *mapelm; + arena_chunk_map_misc_t *miscelm; if (config_stats) nmadvise = 0; @@ -862,13 +866,13 @@ arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms) malloc_mutex_unlock(&arena->lock); - ql_foreach(mapelm, mapelms, dr_link) { + ql_foreach(miscelm, miscelms, dr_link) { arena_chunk_t *chunk; size_t pageind, run_size, npages, flag_unzeroed, i; bool unzeroed; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = arena_mapelm_to_pageind(mapelm); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + pageind = arena_miscelm_to_pageind(miscelm); run_size = arena_mapbits_large_size_get(chunk, pageind); npages = run_size >> LG_PAGE; @@ -908,18 +912,19 @@ arena_purge_stashed(arena_t *arena, arena_chunk_mapelms_t *mapelms) } static void -arena_unstash_purged(arena_t *arena, arena_chunk_mapelms_t *mapelms) +arena_unstash_purged(arena_t *arena, arena_chunk_miscelms_t *miscelms) { - arena_chunk_map_t *mapelm; + arena_chunk_map_misc_t *miscelm; /* Deallocate runs. */ - for (mapelm = ql_first(mapelms); mapelm != NULL; - mapelm = ql_first(mapelms)) { - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - size_t pageind = arena_mapelm_to_pageind(mapelm); + for (miscelm = ql_first(miscelms); miscelm != NULL; + miscelm = ql_first(miscelms)) { + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << LG_PAGE)); - ql_remove(mapelms, mapelm, dr_link); + ql_remove(miscelms, miscelm, dr_link); arena_run_dalloc(arena, run, false, true); } } @@ -928,7 +933,7 @@ void arena_purge(arena_t *arena, bool all) { size_t npurge, npurgeable, npurged; - arena_chunk_mapelms_t purge_list; + arena_chunk_miscelms_t purge_list; if (config_debug) { size_t ndirty = arena_dirty_count(arena); @@ -1180,14 +1185,14 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, static arena_run_t * arena_bin_runs_first(arena_bin_t *bin) { - arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); - if (mapelm != NULL) { + arena_chunk_map_misc_t *miscelm = arena_run_tree_first(&bin->runs); + if (miscelm != NULL) { arena_chunk_t *chunk; size_t pageind; arena_run_t *run; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = arena_mapelm_to_pageind(mapelm); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + pageind = arena_miscelm_to_pageind(miscelm); run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); @@ -1202,11 +1207,11 @@ arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); - assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); + assert(arena_run_tree_search(&bin->runs, miscelm) == NULL); - arena_run_tree_insert(&bin->runs, mapelm); + arena_run_tree_insert(&bin->runs, miscelm); } static void @@ -1214,11 +1219,11 @@ arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); - assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); + assert(arena_run_tree_search(&bin->runs, miscelm) != NULL); - arena_run_tree_remove(&bin->runs, mapelm); + arena_run_tree_remove(&bin->runs, miscelm); } static arena_run_t * @@ -1684,9 +1689,8 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t npages, run_ind, past; assert(run != bin->runcur); - assert(arena_run_tree_search(&bin->runs, - arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) - == NULL); + assert(arena_run_tree_search(&bin->runs, arena_miscelm_get(chunk, + ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) == NULL); binind = arena_bin_index(chunk->arena, run->bin); bin_info = &arena_bin_info[binind]; @@ -1749,7 +1753,7 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, - arena_chunk_map_t *mapelm) + arena_chunk_map_bits_t *bitselm) { size_t pageind; arena_run_t *run; @@ -1761,7 +1765,8 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); bin = run->bin; - binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); + binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, + pageind)); bin_info = &arena_bin_info[binind]; if (config_fill || config_stats) size = bin_info->reg_size; @@ -1784,7 +1789,7 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, - size_t pageind, arena_chunk_map_t *mapelm) + size_t pageind, arena_chunk_map_bits_t *bitselm) { arena_run_t *run; arena_bin_t *bin; @@ -1793,7 +1798,7 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); bin = run->bin; malloc_mutex_lock(&bin->lock); - arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); + arena_dalloc_bin_locked(arena, chunk, ptr, bitselm); malloc_mutex_unlock(&bin->lock); } @@ -1801,15 +1806,15 @@ void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind) { - arena_chunk_map_t *mapelm; + arena_chunk_map_bits_t *bitselm; if (config_debug) { /* arena_ptr_small_binind_get() does extra sanity checking. */ assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)) != BININD_INVALID); } - mapelm = arena_mapp_get(chunk, pageind); - arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); + bitselm = arena_bitselm_get(chunk, pageind); + arena_dalloc_bin(arena, chunk, ptr, pageind, bitselm); } #ifdef JEMALLOC_JET @@ -2268,7 +2273,8 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) * be twice as large in order to maintain alignment. */ if (config_fill && opt_redzone) { - size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - 1); + size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - + 1); if (align_min <= REDZONE_MINSIZE) { bin_info->redzone_size = REDZONE_MINSIZE; pad_size = 0; @@ -2404,13 +2410,17 @@ arena_boot(void) */ map_bias = 0; for (i = 0; i < 3; i++) { - header_size = offsetof(arena_chunk_t, map) + - (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); + header_size = offsetof(arena_chunk_t, map_bits) + + ((sizeof(arena_chunk_map_bits_t) + + sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) != 0); } assert(map_bias > 0); + map_misc_offset = offsetof(arena_chunk_t, map_bits) + + sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); + arena_maxclass = chunksize - (map_bias << LG_PAGE); bin_info_init(); diff --git a/src/chunk.c b/src/chunk.c index 38d02868..874002cf 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -28,6 +28,7 @@ size_t chunksize; size_t chunksize_mask; /* (chunksize - 1). */ size_t chunk_npages; size_t map_bias; +size_t map_misc_offset; size_t arena_maxclass; /* Max size class for arenas. */ /******************************************************************************/ diff --git a/src/tcache.c b/src/tcache.c index 4fbc94cc..f86a46e6 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -118,10 +118,10 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, if (chunk->arena == arena) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - arena_chunk_map_t *mapelm = - arena_mapp_get(chunk, pageind); + arena_chunk_map_bits_t *bitselm = + arena_bitselm_get(chunk, pageind); arena_dalloc_bin_locked(arena, chunk, ptr, - mapelm); + bitselm); } else { /* * This object was allocated via a different @@ -393,9 +393,10 @@ tcache_destroy(tcache_t *tcache) arena_t *arena = chunk->arena; size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> LG_PAGE; - arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + arena_chunk_map_bits_t *bitselm = arena_bitselm_get(chunk, + pageind); - arena_dalloc_bin(arena, chunk, tcache, pageind, mapelm); + arena_dalloc_bin(arena, chunk, tcache, pageind, bitselm); } else if (tcache_size <= tcache_maxclass) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); arena_t *arena = chunk->arena; From c21b05ea09874222266b3e36ceb18765fcb4a00b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 4 Sep 2014 22:27:26 -0700 Subject: [PATCH 086/352] Whitespace cleanups. --- INSTALL | 6 +++--- include/jemalloc/internal/prng.h | 2 +- src/zone.c | 14 +++++++------- test/src/SFMT.c | 20 ++++++++++---------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/INSTALL b/INSTALL index 2df667ca..6c46100e 100644 --- a/INSTALL +++ b/INSTALL @@ -56,7 +56,7 @@ any of the following arguments (not a definitive list) to 'configure': replace the "malloc", "calloc", etc. symbols. --without-export - Don't export public APIs. This can be useful when building jemalloc as a + Don't export public APIs. This can be useful when building jemalloc as a static library, or to avoid exporting public APIs when using the zone allocator on OSX. @@ -96,7 +96,7 @@ any of the following arguments (not a definitive list) to 'configure': --enable-ivsalloc Enable validation code, which verifies that pointers reside within - jemalloc-owned chunks before dereferencing them. This incurs a substantial + jemalloc-owned chunks before dereferencing them. This incurs a substantial performance hit. --disable-stats @@ -148,7 +148,7 @@ any of the following arguments (not a definitive list) to 'configure': Disable support for Valgrind. --disable-zone-allocator - Disable zone allocator for Darwin. This means jemalloc won't be hooked as + Disable zone allocator for Darwin. This means jemalloc won't be hooked as the default allocator on OSX/iOS. --enable-utrace diff --git a/include/jemalloc/internal/prng.h b/include/jemalloc/internal/prng.h index 7b2b0651..c6b17972 100644 --- a/include/jemalloc/internal/prng.h +++ b/include/jemalloc/internal/prng.h @@ -15,7 +15,7 @@ * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints. * * This choice of m has the disadvantage that the quality of the bits is - * proportional to bit position. For example. the lowest bit has a cycle of 2, + * proportional to bit position. For example, the lowest bit has a cycle of 2, * the next has a cycle of 4, etc. For this reason, we prefer to use the upper * bits. * diff --git a/src/zone.c b/src/zone.c index a722287b..c6bd533f 100644 --- a/src/zone.c +++ b/src/zone.c @@ -258,13 +258,13 @@ register_zone(void) /* * On OSX 10.6, having the default purgeable zone appear before * the default zone makes some things crash because it thinks it - * owns the default zone allocated pointers. We thus unregister/ - * re-register it in order to ensure it's always after the - * default zone. On OSX < 10.6, there is no purgeable zone, so - * this does nothing. On OSX >= 10.6, unregistering replaces the - * purgeable zone with the last registered zone above, i.e the - * default zone. Registering it again then puts it at the end, - * obviously after the default zone. + * owns the default zone allocated pointers. We thus + * unregister/re-register it in order to ensure it's always + * after the default zone. On OSX < 10.6, there is no purgeable + * zone, so this does nothing. On OSX >= 10.6, unregistering + * replaces the purgeable zone with the last registered zone + * above, i.e the default zone. Registering it again then puts + * it at the end, obviously after the default zone. */ if (purgeable_zone) { malloc_zone_unregister(purgeable_zone); diff --git a/test/src/SFMT.c b/test/src/SFMT.c index 22a5ac55..80cabe05 100644 --- a/test/src/SFMT.c +++ b/test/src/SFMT.c @@ -463,11 +463,11 @@ uint32_t gen_rand32_range(sfmt_t *ctx, uint32_t limit) { above = 0xffffffffU - (0xffffffffU % limit); while (1) { - ret = gen_rand32(ctx); - if (ret < above) { - ret %= limit; - break; - } + ret = gen_rand32(ctx); + if (ret < above) { + ret %= limit; + break; + } } return ret; } @@ -513,11 +513,11 @@ uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit) { above = KQU(0xffffffffffffffff) - (KQU(0xffffffffffffffff) % limit); while (1) { - ret = gen_rand64(ctx); - if (ret < above) { - ret %= limit; - break; - } + ret = gen_rand64(ctx); + if (ret < above) { + ret %= limit; + break; + } } return ret; } From b718cf77e9917f6ae1995c2e2b219ff4219c9f46 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 7 Sep 2014 14:40:19 -0700 Subject: [PATCH 087/352] Optimize [nmd]alloc() fast paths. Optimize [nmd]alloc() fast paths such that the (flags == 0) case is streamlined, flags decoding only happens to the minimum degree necessary, and no conditionals are repeated. --- include/jemalloc/internal/arena.h | 4 +- .../jemalloc/internal/jemalloc_internal.h.in | 52 ++-- include/jemalloc/internal/private_symbols.txt | 1 - include/jemalloc/internal/size_classes.sh | 3 + src/arena.c | 2 +- src/huge.c | 2 +- src/jemalloc.c | 239 ++++++++++-------- 7 files changed, 172 insertions(+), 131 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 986bea92..166d0523 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -577,7 +577,7 @@ small_bin2size_lookup(size_t binind) assert(binind < NBINS); { - size_t ret = ((size_t)(small_bin2size_tab[binind])); + size_t ret = (size_t)small_bin2size_tab[binind]; assert(ret == small_bin2size_compute(binind)); return (ret); } @@ -615,7 +615,7 @@ small_s2u_compute(size_t size) JEMALLOC_ALWAYS_INLINE size_t small_s2u_lookup(size_t size) { - size_t ret = (small_bin2size(small_size2bin(size))); + size_t ret = small_bin2size(small_size2bin(size)); assert(ret == small_s2u_compute(size)); return (ret); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 1c2f3d44..59ae8d55 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -165,7 +165,17 @@ static const bool config_ivsalloc = #include "jemalloc/internal/jemalloc_internal_macros.h" +#define MALLOCX_ARENA_MASK ((int)~0xff) #define MALLOCX_LG_ALIGN_MASK ((int)0x3f) +/* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ +#define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ + (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK)) +#define MALLOCX_ALIGN_GET(flags) \ + (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1)) +#define MALLOCX_ZERO_GET(flags) \ + ((bool)(flags & MALLOCX_ZERO)) +#define MALLOCX_ARENA_GET(flags) \ + (((unsigned)(flags >> 8)) - 1) /* Smallest size class to support. */ #define LG_TINY_MIN 3 @@ -625,15 +635,13 @@ size_t u2rz(size_t usize); size_t p2rz(const void *ptr); void idalloct(void *ptr, bool try_tcache); void idalloc(void *ptr); -void iqalloct(void *ptr, bool try_tcache); -void iqalloc(void *ptr); +void iqalloc(void *ptr, bool try_tcache); void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); -void *iralloct(void *ptr, size_t size, size_t extra, size_t alignment, - bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); -void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, - bool zero); +void *iralloct(void *ptr, size_t size, size_t alignment, bool zero, + bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); +void *iralloc(void *ptr, size_t size, size_t alignment, bool zero); bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero); malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) @@ -787,7 +795,7 @@ idalloc(void *ptr) } JEMALLOC_ALWAYS_INLINE void -iqalloct(void *ptr, bool try_tcache) +iqalloc(void *ptr, bool try_tcache) { if (config_fill && opt_quarantine) @@ -796,13 +804,6 @@ iqalloct(void *ptr, bool try_tcache) idalloct(ptr, try_tcache); } -JEMALLOC_ALWAYS_INLINE void -iqalloc(void *ptr) -{ - - iqalloct(ptr, true); -} - JEMALLOC_ALWAYS_INLINE void * iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, @@ -832,12 +833,12 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - iqalloct(ptr, try_tcache_dalloc); + iqalloc(ptr, try_tcache_dalloc); return (p); } JEMALLOC_ALWAYS_INLINE void * -iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, +iralloct(void *ptr, size_t size, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) { size_t oldsize; @@ -853,25 +854,24 @@ iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, * Existing object alignment is inadequate; allocate new space * and copy. */ - return (iralloct_realign(ptr, oldsize, size, extra, alignment, - zero, try_tcache_alloc, try_tcache_dalloc, arena)); + return (iralloct_realign(ptr, oldsize, size, 0, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena)); } - if (size + extra <= arena_maxclass) { - return (arena_ralloc(arena, ptr, oldsize, size, extra, - alignment, zero, try_tcache_alloc, - try_tcache_dalloc)); + if (size <= arena_maxclass) { + return (arena_ralloc(arena, ptr, oldsize, size, 0, alignment, + zero, try_tcache_alloc, try_tcache_dalloc)); } else { - return (huge_ralloc(arena, ptr, oldsize, size, extra, - alignment, zero, try_tcache_dalloc)); + return (huge_ralloc(arena, ptr, oldsize, size, 0, alignment, + zero, try_tcache_dalloc)); } } JEMALLOC_ALWAYS_INLINE void * -iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) +iralloc(void *ptr, size_t size, size_t alignment, bool zero) { - return (iralloct(ptr, size, extra, alignment, zero, true, true, NULL)); + return (iralloct(ptr, size, alignment, zero, true, true, NULL)); } JEMALLOC_ALWAYS_INLINE bool diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 9ca139ab..84f05910 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -224,7 +224,6 @@ in_valgrind ipalloc ipalloct iqalloc -iqalloct iralloc iralloct iralloct_realign diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 379d36c2..0cfac72d 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -202,6 +202,7 @@ cat <deallocated += usize; if (config_valgrind && in_valgrind) rzsize = p2rz(ptr); - iqalloc(ptr); + iqalloc(ptr, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); } @@ -1236,7 +1235,7 @@ je_realloc(void *ptr, size_t size) if (ptr != NULL) { /* realloc(ptr, 0) is equivalent to free(ptr). */ UTRACE(ptr, 0, 0); - ifree(ptr); + ifree(ptr, true); return (NULL); } size = 1; @@ -1261,7 +1260,7 @@ je_realloc(void *ptr, size_t size) } else { if (config_stats || (config_valgrind && in_valgrind)) usize = s2u(size); - ret = iralloc(ptr, size, 0, 0, false); + ret = iralloc(ptr, size, 0, false); } } else { /* realloc(NULL, size) is equivalent to malloc(size). */ @@ -1295,7 +1294,7 @@ je_free(void *ptr) UTRACE(ptr, 0, 0); if (ptr != NULL) - ifree(ptr); + ifree(ptr, true); } /* @@ -1363,99 +1362,153 @@ JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = * Begin non-standard functions. */ +JEMALLOC_ALWAYS_INLINE_C void +imallocx_flags_decode_hard(size_t size, int flags, size_t *usize, + size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) +{ + + if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { + *alignment = 0; + *usize = s2u(size); + } else { + *alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); + *usize = sa2u(size, *alignment); + } + *zero = MALLOCX_ZERO_GET(flags); + if ((flags & MALLOCX_ARENA_MASK) != 0) { + unsigned arena_ind = MALLOCX_ARENA_GET(flags); + *try_tcache = false; + *arena = arenas[arena_ind]; + } else { + *try_tcache = true; + *arena = NULL; + } +} + +JEMALLOC_ALWAYS_INLINE_C void +imallocx_flags_decode(size_t size, int flags, size_t *usize, size_t *alignment, + bool *zero, bool *try_tcache, arena_t **arena) +{ + + if (flags == 0) { + *usize = s2u(size); + assert(usize != 0); + *alignment = 0; + *zero = false; + *try_tcache = true; + *arena = NULL; + } else { + imallocx_flags_decode_hard(size, flags, usize, alignment, zero, + try_tcache, arena); + } +} + JEMALLOC_ALWAYS_INLINE_C void * -imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, +imallocx_flags(size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena) { - assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, - alignment))); - if (alignment != 0) return (ipalloct(usize, alignment, zero, try_tcache, arena)); - else if (zero) + if (zero) return (icalloct(usize, try_tcache, arena)); - else - return (imalloct(usize, try_tcache, arena)); + return (imalloct(usize, try_tcache, arena)); +} + + +JEMALLOC_ALWAYS_INLINE_C void * +imallocx_maybe_flags(size_t size, int flags, size_t usize, size_t alignment, + bool zero, bool try_tcache, arena_t *arena) +{ + + if (flags == 0) + return (imalloc(size)); + return (imallocx_flags(usize, alignment, zero, try_tcache, arena)); } static void * -imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena, prof_tctx_t *tctx) +imallocx_prof_sample(size_t size, int flags, size_t usize, size_t alignment, + bool zero, bool try_tcache, arena_t *arena) { void *p; - if (tctx == NULL) - return (NULL); if (usize <= SMALL_MAXCLASS) { - size_t usize_promoted = (alignment == 0) ? - s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); - assert(usize_promoted != 0); - p = imallocx(usize_promoted, alignment, zero, try_tcache, - arena); + assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : + sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); + p = imalloc(LARGE_MINCLASS); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); - } else - p = imallocx(usize, alignment, zero, try_tcache, arena); + } else { + p = imallocx_maybe_flags(size, flags, usize, alignment, zero, + try_tcache, arena); + } return (p); } JEMALLOC_ALWAYS_INLINE_C void * -imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena, prof_tctx_t *tctx) +imallocx_prof(size_t size, int flags, size_t *usize) { void *p; + size_t alignment; + bool zero; + bool try_tcache; + arena_t *arena; + prof_tctx_t *tctx; - if ((uintptr_t)tctx != (uintptr_t)1U) { - p = imallocx_prof_sample(usize, alignment, zero, try_tcache, - arena, tctx); + imallocx_flags_decode(size, flags, usize, &alignment, &zero, + &try_tcache, &arena); + tctx = prof_alloc_prep(*usize); + if ((uintptr_t)tctx == (uintptr_t)1U) { + p = imallocx_maybe_flags(size, flags, *usize, alignment, zero, + try_tcache, arena); + } else if ((uintptr_t)tctx > (uintptr_t)1U) { + p = imallocx_prof_sample(size, flags, *usize, alignment, zero, + try_tcache, arena); } else - p = imallocx(usize, alignment, zero, try_tcache, arena); + p = NULL; if (p == NULL) return (NULL); - prof_malloc(p, usize, tctx); + prof_malloc(p, *usize, tctx); return (p); } +JEMALLOC_ALWAYS_INLINE_C void * +imallocx_no_prof(size_t size, int flags, size_t *usize) +{ + size_t alignment; + bool zero; + bool try_tcache; + arena_t *arena; + + if (flags == 0) { + if (config_stats || (config_valgrind && in_valgrind)) + *usize = s2u(size); + return (imalloc(size)); + } + + imallocx_flags_decode_hard(size, flags, usize, &alignment, &zero, + &try_tcache, &arena); + return (imallocx_flags(*usize, alignment, zero, try_tcache, arena)); +} + void * je_mallocx(size_t size, int flags) { void *p; size_t usize; - size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) - & (SIZE_T_MAX-1)); - bool zero = flags & MALLOCX_ZERO; - unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; - arena_t *arena; - bool try_tcache; assert(size != 0); if (malloc_init()) goto label_oom; - if (arena_ind != UINT_MAX) { - arena = arenas[arena_ind]; - try_tcache = false; - } else { - arena = NULL; - try_tcache = true; - } - - usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); - assert(usize != 0); - - if (config_prof && opt_prof) { - prof_tctx_t *tctx; - - tctx = prof_alloc_prep(usize); - p = imallocx_prof(usize, alignment, zero, try_tcache, arena, - tctx); - } else - p = imallocx(usize, alignment, zero, try_tcache, arena); + if (config_prof && opt_prof) + p = imallocx_prof(size, flags, &usize); + else + p = imallocx_no_prof(size, flags, &usize); if (p == NULL) goto label_oom; @@ -1464,7 +1517,7 @@ je_mallocx(size_t size, int flags) thread_allocated_tsd_get()->allocated += usize; } UTRACE(0, size, p); - JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); + JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags)); return (p); label_oom: if (config_xmalloc && opt_xmalloc) { @@ -1485,15 +1538,14 @@ irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= - size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero, + p = iralloct(oldptr, LARGE_MINCLASS, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else { - p = iralloct(oldptr, size, 0, alignment, zero, - try_tcache_alloc, try_tcache_dalloc, arena); + p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc, + try_tcache_dalloc, arena); } return (p); @@ -1512,8 +1564,8 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx); else { - p = iralloct(oldptr, size, 0, alignment, zero, - try_tcache_alloc, try_tcache_dalloc, arena); + p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc, + try_tcache_dalloc, arena); } if (p == NULL) return (NULL); @@ -1540,10 +1592,8 @@ je_rallocx(void *ptr, size_t size, int flags) void *p; size_t usize, old_usize; UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); - size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) - & (SIZE_T_MAX-1)); + size_t alignment = MALLOCX_ALIGN_GET(flags); bool zero = flags & MALLOCX_ZERO; - unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; bool try_tcache_alloc, try_tcache_dalloc; arena_t *arena; @@ -1552,7 +1602,8 @@ je_rallocx(void *ptr, size_t size, int flags) assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); - if (arena_ind != UINT_MAX) { + if ((flags & MALLOCX_ARENA_MASK) != 0) { + unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk; try_tcache_alloc = false; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); @@ -1582,7 +1633,7 @@ je_rallocx(void *ptr, size_t size, int flags) if (p == NULL) goto label_oom; } else { - p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc, + p = iralloct(ptr, size, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); if (p == NULL) goto label_oom; @@ -1677,10 +1728,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { size_t usize, old_usize; UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); - size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) - & (SIZE_T_MAX-1)); + size_t alignment = MALLOCX_ALIGN_GET(flags); bool zero = flags & MALLOCX_ZERO; - unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; arena_t *arena; assert(ptr != NULL); @@ -1689,9 +1738,10 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); - if (arena_ind != UINT_MAX) + if ((flags & MALLOCX_ARENA_MASK) != 0) { + unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena = arenas[arena_ind]; - else + } else arena = NULL; old_usize = isalloc(ptr, config_prof); @@ -1753,15 +1803,13 @@ je_sallocx(const void *ptr, int flags) void je_dallocx(void *ptr, int flags) { - size_t usize; - UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; bool try_tcache; assert(ptr != NULL); assert(malloc_initialized || IS_INITIALIZER); - if (arena_ind != UINT_MAX) { + if ((flags & MALLOCX_ARENA_MASK) != 0) { + unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); try_tcache = (chunk == ptr || chunk->arena != arenas[arena_ind]); @@ -1769,34 +1817,25 @@ je_dallocx(void *ptr, int flags) try_tcache = true; UTRACE(ptr, 0, 0); - if (config_stats || config_valgrind) - usize = isalloc(ptr, config_prof); - if (config_prof && opt_prof) { - if (config_stats == false && config_valgrind == false) - usize = isalloc(ptr, config_prof); - prof_free(ptr, usize); - } - if (config_stats) - thread_allocated_tsd_get()->deallocated += usize; - if (config_valgrind && in_valgrind) - rzsize = p2rz(ptr); - iqalloct(ptr, try_tcache); - JEMALLOC_VALGRIND_FREE(ptr, rzsize); + ifree(ptr, try_tcache); } size_t je_nallocx(size_t size, int flags) { size_t usize; - size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) - & (SIZE_T_MAX-1)); assert(size != 0); if (malloc_init()) return (0); - usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); + if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) + usize = s2u(size); + else { + size_t alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); + usize = sa2u(size, alignment); + } assert(usize != 0); return (usize); } From 82e88d1ecfe3d7bf700355cb5023ab61559f9578 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 7 Sep 2014 19:55:03 -0700 Subject: [PATCH 088/352] Move typedefs from jemalloc_protos.h.in to jemalloc_typedefs.h.in. Move typedefs from jemalloc_protos.h.in to jemalloc_typedefs.h.in, so that typedefs aren't redefined when compiling stress tests. --- .gitignore | 1 + configure.ac | 3 +++ include/jemalloc/jemalloc.sh | 2 +- include/jemalloc/jemalloc_protos.h.in | 3 --- include/jemalloc/jemalloc_typedefs.h.in | 2 ++ 5 files changed, 7 insertions(+), 4 deletions(-) create mode 100644 include/jemalloc/jemalloc_typedefs.h.in diff --git a/.gitignore b/.gitignore index ec9c0b92..79d454f2 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ /include/jemalloc/jemalloc_protos.h /include/jemalloc/jemalloc_protos_jet.h /include/jemalloc/jemalloc_rename.h +/include/jemalloc/jemalloc_typedefs.h /src/*.[od] /src/*.gcda diff --git a/configure.ac b/configure.ac index 3b658852..ce4af213 100644 --- a/configure.ac +++ b/configure.ac @@ -545,6 +545,7 @@ cfgoutputs_in="${cfgoutputs_in} doc/manpages.xsl.in" cfgoutputs_in="${cfgoutputs_in} doc/jemalloc.xml.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_macros.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_protos.h.in" +cfgoutputs_in="${cfgoutputs_in} include/jemalloc/jemalloc_typedefs.h.in" cfgoutputs_in="${cfgoutputs_in} include/jemalloc/internal/jemalloc_internal.h.in" cfgoutputs_in="${cfgoutputs_in} test/test.sh.in" cfgoutputs_in="${cfgoutputs_in} test/include/test/jemalloc_test.h.in" @@ -555,6 +556,7 @@ cfgoutputs_out="${cfgoutputs_out} doc/manpages.xsl" cfgoutputs_out="${cfgoutputs_out} doc/jemalloc.xml" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_macros.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_protos.h" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc_typedefs.h" cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal.h" cfgoutputs_out="${cfgoutputs_out} test/test.sh" cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h" @@ -565,6 +567,7 @@ cfgoutputs_tup="${cfgoutputs_tup} doc/manpages.xsl:doc/manpages.xsl.in" cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc.xml:doc/jemalloc.xml.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_macros.h:include/jemalloc/jemalloc_macros.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_protos.h:include/jemalloc/jemalloc_protos.h.in" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc_typedefs.h:include/jemalloc/jemalloc_typedefs.h.in" cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" cfgoutputs_tup="${cfgoutputs_tup} test/test.sh:test/test.sh.in" cfgoutputs_tup="${cfgoutputs_tup} test/include/test/jemalloc_test.h:test/include/test/jemalloc_test.h.in" diff --git a/include/jemalloc/jemalloc.sh b/include/jemalloc/jemalloc.sh index e4738eba..7e1c8be1 100755 --- a/include/jemalloc/jemalloc.sh +++ b/include/jemalloc/jemalloc.sh @@ -12,7 +12,7 @@ extern "C" { EOF for hdr in jemalloc_defs.h jemalloc_rename.h jemalloc_macros.h \ - jemalloc_protos.h jemalloc_mangle.h ; do + jemalloc_protos.h jemalloc_typedefs.h jemalloc_mangle.h ; do cat "${objroot}include/jemalloc/${hdr}" \ | grep -v 'Generated from .* by configure\.' \ | sed -e 's/^#define /#define /g' \ diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 67268c47..59aeee11 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -44,6 +44,3 @@ JEMALLOC_EXPORT void * @je_@memalign(size_t alignment, size_t size) #ifdef JEMALLOC_OVERRIDE_VALLOC JEMALLOC_EXPORT void * @je_@valloc(size_t size) JEMALLOC_ATTR(malloc); #endif - -typedef void *(chunk_alloc_t)(size_t, size_t, bool *, unsigned); -typedef bool (chunk_dalloc_t)(void *, size_t, unsigned); diff --git a/include/jemalloc/jemalloc_typedefs.h.in b/include/jemalloc/jemalloc_typedefs.h.in new file mode 100644 index 00000000..47e57ca7 --- /dev/null +++ b/include/jemalloc/jemalloc_typedefs.h.in @@ -0,0 +1,2 @@ +typedef void *(chunk_alloc_t)(size_t, size_t, bool *, unsigned); +typedef bool (chunk_dalloc_t)(void *, size_t, unsigned); From b67ec3c4973e8f7ca272c13472aa98c8a3ba4de4 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 7 Sep 2014 19:57:24 -0700 Subject: [PATCH 089/352] Add a simple timer implementation for use in benchmarking. --- Makefile.in | 2 +- test/include/test/jemalloc_test.h.in | 2 + test/include/test/timer.h | 15 ++++++++ test/src/timer.c | 57 ++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 test/include/test/timer.h create mode 100644 test/src/timer.c diff --git a/Makefile.in b/Makefile.in index b5f0ee90..d3e91b56 100644 --- a/Makefile.in +++ b/Makefile.in @@ -108,7 +108,7 @@ DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3) DOCS := $(DOCS_HTML) $(DOCS_MAN3) C_TESTLIB_SRCS := $(srcroot)test/src/math.c $(srcroot)test/src/mtx.c \ $(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \ - $(srcroot)test/src/thd.c + $(srcroot)test/src/thd.c $(srcroot)test/src/timer.c C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c TESTS_UNIT := $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/bitmap.c \ diff --git a/test/include/test/jemalloc_test.h.in b/test/include/test/jemalloc_test.h.in index 730a55db..a93c4f67 100644 --- a/test/include/test/jemalloc_test.h.in +++ b/test/include/test/jemalloc_test.h.in @@ -5,6 +5,7 @@ #include #include #include +#include #ifdef _WIN32 # include @@ -136,6 +137,7 @@ #include "test/mtx.h" #include "test/mq.h" #include "test/test.h" +#include "test/timer.h" #include "test/thd.h" #define MEXP 19937 #include "test/SFMT.h" diff --git a/test/include/test/timer.h b/test/include/test/timer.h new file mode 100644 index 00000000..f21ccf10 --- /dev/null +++ b/test/include/test/timer.h @@ -0,0 +1,15 @@ +/* + * Simple timer, for use in benchmark reporting. + */ + +#include + +typedef struct { + struct timeval tv0; + struct timeval tv1; +} timer_t; + +void timer_start(timer_t *timer); +void timer_stop(timer_t *timer); +uint64_t timer_usec(const timer_t *timer); +void timer_ratio(timer_t *a, timer_t *b, char *buf, size_t buflen); diff --git a/test/src/timer.c b/test/src/timer.c new file mode 100644 index 00000000..17ead172 --- /dev/null +++ b/test/src/timer.c @@ -0,0 +1,57 @@ +#include "test/jemalloc_test.h" + +void +timer_start(timer_t *timer) +{ + + gettimeofday(&timer->tv0, NULL); +} + +void +timer_stop(timer_t *timer) +{ + + gettimeofday(&timer->tv1, NULL); +} + +uint64_t +timer_usec(const timer_t *timer) +{ + + return (((timer->tv1.tv_sec - timer->tv0.tv_sec) * 1000000) + + timer->tv1.tv_usec - timer->tv0.tv_usec); +} + +void +timer_ratio(timer_t *a, timer_t *b, char *buf, size_t buflen) +{ + uint64_t t0 = timer_usec(a); + uint64_t t1 = timer_usec(b); + uint64_t mult; + unsigned i = 0; + unsigned j; + int n; + + /* Whole. */ + n = malloc_snprintf(&buf[i], buflen-i, "%"PRIu64, t0 / t1); + i += n; + if (i >= buflen) + return; + mult = 1; + for (j = 0; j < n; j++) + mult *= 10; + + /* Decimal. */ + n = malloc_snprintf(&buf[i], buflen-i, "."); + i += n; + + /* Fraction. */ + while (i < buflen-1) { + uint64_t round = (i+1 == buflen-1 && ((t0 * mult * 10 / t1) % 10 + >= 5)) ? 1 : 0; + n = malloc_snprintf(&buf[i], buflen-i, + "%"PRIu64, (t0 * mult / t1) % 10 + round); + i += n; + mult *= 10; + } +} From 423d78a21bc6c9a038bdf436ad2cee194560d488 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 7 Sep 2014 19:58:04 -0700 Subject: [PATCH 090/352] Add microbench tests. --- Makefile.in | 2 +- test/stress/microbench.c | 142 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 test/stress/microbench.c diff --git a/Makefile.in b/Makefile.in index d3e91b56..1446dbe6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -144,7 +144,7 @@ TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/thread_tcache_enabled.c \ $(srcroot)test/integration/xallocx.c \ $(srcroot)test/integration/chunk.c -TESTS_STRESS := +TESTS_STRESS := $(srcroot)test/stress/microbench.c TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_STRESS) C_OBJS := $(C_SRCS:$(srcroot)%.c=$(objroot)%.$(O)) diff --git a/test/stress/microbench.c b/test/stress/microbench.c new file mode 100644 index 00000000..8c252153 --- /dev/null +++ b/test/stress/microbench.c @@ -0,0 +1,142 @@ +#include "test/jemalloc_test.h" + +JEMALLOC_INLINE_C void +time_func(timer_t *timer, uint64_t nwarmup, uint64_t niter, void (*func)(void)) +{ + uint64_t i; + + for (i = 0; i < nwarmup; i++) + func(); + timer_start(timer); + for (i = 0; i < niter; i++) + func(); + timer_stop(timer); +} + +void +compare_funcs(uint64_t nwarmup, uint64_t niter, const char *name_a, + void (*func_a), const char *name_b, void (*func_b)) +{ + timer_t timer_a, timer_b; + char ratio_buf[6]; + + time_func(&timer_a, nwarmup, niter, func_a); + time_func(&timer_b, nwarmup, niter, func_b); + + timer_ratio(&timer_a, &timer_b, ratio_buf, sizeof(ratio_buf)); + malloc_printf("%"PRIu64" iterations, %s=%"PRIu64"us, " + "%s=%"PRIu64"us, ratio=1:%s\n", + niter, name_a, timer_usec(&timer_a), name_b, timer_usec(&timer_b), + ratio_buf); +} + +static void +malloc_vs_mallocx_malloc(void) +{ + + free(malloc(1)); +} + +static void +malloc_vs_mallocx_mallocx(void) +{ + + free(mallocx(1, 0)); +} + +TEST_BEGIN(test_malloc_vs_mallocx) +{ + + compare_funcs(10*1000*1000, 100*1000*1000, "malloc", + malloc_vs_mallocx_malloc, "mallocx", malloc_vs_mallocx_mallocx); +} +TEST_END + +static void +free_vs_dallocx_free(void) +{ + + free(malloc(1)); +} + +static void +free_vs_dallocx_dallocx(void) +{ + + dallocx(malloc(1), 0); +} + +TEST_BEGIN(test_free_vs_dallocx) +{ + + compare_funcs(10*1000*1000, 100*1000*1000, "free", free_vs_dallocx_free, + "dallocx", free_vs_dallocx_dallocx); +} +TEST_END + +static void +mus_vs_sallocx_mus(void) +{ + void *p; + + p = malloc(1); + malloc_usable_size(p); + free(p); +} + +static void +mus_vs_sallocx_sallocx(void) +{ + void *p; + + p = malloc(1); + sallocx(p, 0); + free(p); +} + +TEST_BEGIN(test_mus_vs_sallocx) +{ + + compare_funcs(10*1000*1000, 100*1000*1000, "malloc_usable_size", + mus_vs_sallocx_mus, "sallocx", mus_vs_sallocx_sallocx); +} +TEST_END + +static void +sallocx_vs_nallocx_sallocx(void) +{ + void *p; + + p = malloc(1); + sallocx(p, 0); + free(p); +} + +static void +sallocx_vs_nallocx_nallocx(void) +{ + void *p; + + p = malloc(1); + nallocx(1, 0); + free(p); +} + +TEST_BEGIN(test_sallocx_vs_nallocx) +{ + + compare_funcs(10*1000*1000, 100*1000*1000, "sallocx", + sallocx_vs_nallocx_sallocx, "nallocx", sallocx_vs_nallocx_nallocx); +} +TEST_END + +int +main(void) +{ + + return (test( + test_malloc_vs_mallocx, + test_free_vs_dallocx, + test_mus_vs_sallocx, + test_sallocx_vs_nallocx)); +} From c3bfe9569a9927dc881b6d8ac025c423d66a541f Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Mon, 8 Sep 2014 00:46:12 -0400 Subject: [PATCH 091/352] avoid conflict with the POSIX timer_t type It hits a compilation error with glibc 2.19 without a rename. --- test/include/test/timer.h | 10 +++++----- test/src/timer.c | 8 ++++---- test/stress/microbench.c | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/test/include/test/timer.h b/test/include/test/timer.h index f21ccf10..6877e4ac 100644 --- a/test/include/test/timer.h +++ b/test/include/test/timer.h @@ -7,9 +7,9 @@ typedef struct { struct timeval tv0; struct timeval tv1; -} timer_t; +} timedelta_t; -void timer_start(timer_t *timer); -void timer_stop(timer_t *timer); -uint64_t timer_usec(const timer_t *timer); -void timer_ratio(timer_t *a, timer_t *b, char *buf, size_t buflen); +void timer_start(timedelta_t *timer); +void timer_stop(timedelta_t *timer); +uint64_t timer_usec(const timedelta_t *timer); +void timer_ratio(timedelta_t *a, timedelta_t *b, char *buf, size_t buflen); diff --git a/test/src/timer.c b/test/src/timer.c index 17ead172..36fbedd4 100644 --- a/test/src/timer.c +++ b/test/src/timer.c @@ -1,21 +1,21 @@ #include "test/jemalloc_test.h" void -timer_start(timer_t *timer) +timer_start(timedelta_t *timer) { gettimeofday(&timer->tv0, NULL); } void -timer_stop(timer_t *timer) +timer_stop(timedelta_t *timer) { gettimeofday(&timer->tv1, NULL); } uint64_t -timer_usec(const timer_t *timer) +timer_usec(const timedelta_t *timer) { return (((timer->tv1.tv_sec - timer->tv0.tv_sec) * 1000000) + @@ -23,7 +23,7 @@ timer_usec(const timer_t *timer) } void -timer_ratio(timer_t *a, timer_t *b, char *buf, size_t buflen) +timer_ratio(timedelta_t *a, timedelta_t *b, char *buf, size_t buflen) { uint64_t t0 = timer_usec(a); uint64_t t1 = timer_usec(b); diff --git a/test/stress/microbench.c b/test/stress/microbench.c index 8c252153..616f361c 100644 --- a/test/stress/microbench.c +++ b/test/stress/microbench.c @@ -1,7 +1,7 @@ #include "test/jemalloc_test.h" JEMALLOC_INLINE_C void -time_func(timer_t *timer, uint64_t nwarmup, uint64_t niter, void (*func)(void)) +time_func(timedelta_t *timer, uint64_t nwarmup, uint64_t niter, void (*func)(void)) { uint64_t i; @@ -17,7 +17,7 @@ void compare_funcs(uint64_t nwarmup, uint64_t niter, const char *name_a, void (*func_a), const char *name_b, void (*func_b)) { - timer_t timer_a, timer_b; + timedelta_t timer_a, timer_b; char ratio_buf[6]; time_func(&timer_a, nwarmup, niter, func_a); From a1f3929ffd1bd958734a2747cf2000a9b2a5db0b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 8 Sep 2014 16:23:48 -0700 Subject: [PATCH 092/352] Thwart optimization of free(malloc(1)) in microbench. --- test/stress/microbench.c | 44 +++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/test/stress/microbench.c b/test/stress/microbench.c index 616f361c..8e1017cc 100644 --- a/test/stress/microbench.c +++ b/test/stress/microbench.c @@ -31,46 +31,52 @@ compare_funcs(uint64_t nwarmup, uint64_t niter, const char *name_a, } static void -malloc_vs_mallocx_malloc(void) +malloc_free(void) { - - free(malloc(1)); + /* The compiler can optimize away free(malloc(1))! */ + void *p = malloc(1); + if (p == NULL) { + test_fail("Unexpected malloc() failure"); + return; + } + free(p); } static void -malloc_vs_mallocx_mallocx(void) +mallocx_free(void) { - - free(mallocx(1, 0)); + void *p = mallocx(1, 0); + if (p == NULL) { + test_fail("Unexpected mallocx() failure"); + return; + } + free(p); } TEST_BEGIN(test_malloc_vs_mallocx) { compare_funcs(10*1000*1000, 100*1000*1000, "malloc", - malloc_vs_mallocx_malloc, "mallocx", malloc_vs_mallocx_mallocx); + malloc_free, "mallocx", mallocx_free); } TEST_END static void -free_vs_dallocx_free(void) +malloc_dallocx(void) { - - free(malloc(1)); -} - -static void -free_vs_dallocx_dallocx(void) -{ - - dallocx(malloc(1), 0); + void *p = malloc(1); + if (p == NULL) { + test_fail("Unexpected malloc() failure"); + return; + } + dallocx(p, 0); } TEST_BEGIN(test_free_vs_dallocx) { - compare_funcs(10*1000*1000, 100*1000*1000, "free", free_vs_dallocx_free, - "dallocx", free_vs_dallocx_dallocx); + compare_funcs(10*1000*1000, 100*1000*1000, "free", malloc_free, + "dallocx", malloc_dallocx); } TEST_END From c3f865074923bf388742da3ec52dca857a0960a2 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 8 Sep 2014 16:47:51 -0700 Subject: [PATCH 093/352] Add relevant function attributes to [msn]allocx(). --- include/jemalloc/jemalloc_protos.h.in | 9 ++++++--- test/stress/microbench.c | 26 +++++++++----------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index 59aeee11..b365eb4a 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -17,13 +17,16 @@ JEMALLOC_EXPORT void *@je_@aligned_alloc(size_t alignment, size_t size) JEMALLOC_EXPORT void *@je_@realloc(void *ptr, size_t size); JEMALLOC_EXPORT void @je_@free(void *ptr); -JEMALLOC_EXPORT void *@je_@mallocx(size_t size, int flags); +JEMALLOC_EXPORT void *@je_@mallocx(size_t size, int flags) + JEMALLOC_ATTR(malloc); JEMALLOC_EXPORT void *@je_@rallocx(void *ptr, size_t size, int flags); JEMALLOC_EXPORT size_t @je_@xallocx(void *ptr, size_t size, size_t extra, int flags); -JEMALLOC_EXPORT size_t @je_@sallocx(const void *ptr, int flags); +JEMALLOC_EXPORT size_t @je_@sallocx(const void *ptr, int flags) + JEMALLOC_ATTR(pure); JEMALLOC_EXPORT void @je_@dallocx(void *ptr, int flags); -JEMALLOC_EXPORT size_t @je_@nallocx(size_t size, int flags); +JEMALLOC_EXPORT size_t @je_@nallocx(size_t size, int flags) + JEMALLOC_ATTR(pure); JEMALLOC_EXPORT int @je_@mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); diff --git a/test/stress/microbench.c b/test/stress/microbench.c index 8e1017cc..60c02db3 100644 --- a/test/stress/microbench.c +++ b/test/stress/microbench.c @@ -81,7 +81,7 @@ TEST_BEGIN(test_free_vs_dallocx) TEST_END static void -mus_vs_sallocx_mus(void) +malloc_mus_free(void) { void *p; @@ -91,12 +91,13 @@ mus_vs_sallocx_mus(void) } static void -mus_vs_sallocx_sallocx(void) +malloc_sallocx_free(void) { void *p; p = malloc(1); - sallocx(p, 0); + if (sallocx(p, 0) < 1) + test_fail("Unexpected sallocx() failure"); free(p); } @@ -104,27 +105,18 @@ TEST_BEGIN(test_mus_vs_sallocx) { compare_funcs(10*1000*1000, 100*1000*1000, "malloc_usable_size", - mus_vs_sallocx_mus, "sallocx", mus_vs_sallocx_sallocx); + malloc_mus_free, "sallocx", malloc_sallocx_free); } TEST_END static void -sallocx_vs_nallocx_sallocx(void) +malloc_nallocx_free(void) { void *p; p = malloc(1); - sallocx(p, 0); - free(p); -} - -static void -sallocx_vs_nallocx_nallocx(void) -{ - void *p; - - p = malloc(1); - nallocx(1, 0); + if (nallocx(1, 0) < 1) + test_fail("Unexpected nallocx() failure"); free(p); } @@ -132,7 +124,7 @@ TEST_BEGIN(test_sallocx_vs_nallocx) { compare_funcs(10*1000*1000, 100*1000*1000, "sallocx", - sallocx_vs_nallocx_sallocx, "nallocx", sallocx_vs_nallocx_nallocx); + malloc_sallocx_free, "nallocx", malloc_nallocx_free); } TEST_END From 4cfe55166e0173be745c53adb0fecf50d11d1227 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Thu, 28 Aug 2014 15:41:48 -0400 Subject: [PATCH 094/352] Add support for sized deallocation. This adds a new `sdallocx` function to the external API, allowing the size to be passed by the caller. It avoids some extra reads in the thread cache fast path. In the case where stats are enabled, this avoids the work of calculating the size from the pointer. An assertion validates the size that's passed in, so enabling debugging will allow users of the API to debug cases where an incorrect size is passed in. The performance win for a contrived microbenchmark doing an allocation and immediately freeing it is ~10%. It may have a different impact on a real workload. Closes #28 --- Makefile.in | 1 + configure.ac | 2 +- doc/jemalloc.xml.in | 19 ++++++- include/jemalloc/internal/arena.h | 33 ++++++++++- .../jemalloc/internal/jemalloc_internal.h.in | 26 +++++++++ include/jemalloc/internal/private_symbols.txt | 3 + include/jemalloc/jemalloc_protos.h.in | 1 + src/jemalloc.c | 44 ++++++++++++++ test/integration/sdallocx.c | 57 +++++++++++++++++++ test/stress/microbench.c | 20 +++++++ 10 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 test/integration/sdallocx.c diff --git a/Makefile.in b/Makefile.in index 1446dbe6..ac56d8fa 100644 --- a/Makefile.in +++ b/Makefile.in @@ -136,6 +136,7 @@ TESTS_UNIT_AUX := $(srcroot)test/unit/prof_accum_a.c \ $(srcroot)test/unit/prof_accum_b.c TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/allocated.c \ + $(srcroot)test/integration/sdallocx.c \ $(srcroot)test/integration/mallocx.c \ $(srcroot)test/integration/MALLOCX_ARENA.c \ $(srcroot)test/integration/posix_memalign.c \ diff --git a/configure.ac b/configure.ac index ce4af213..d221876c 100644 --- a/configure.ac +++ b/configure.ac @@ -452,7 +452,7 @@ AC_PROG_RANLIB AC_PATH_PROG([LD], [ld], [false], [$PATH]) AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH]) -public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free mallocx rallocx xallocx sallocx dallocx nallocx mallctl mallctlnametomib mallctlbymib malloc_stats_print malloc_usable_size" +public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free mallocx rallocx xallocx sallocx dallocx sdallocx nallocx mallctl mallctlnametomib mallctlbymib malloc_stats_print malloc_usable_size" dnl Check for allocator-related functions that should be wrapped. AC_CHECK_FUNC([memalign], diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 8f4327f3..e5c229fe 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -38,6 +38,7 @@ xallocx sallocx dallocx + sdallocx nallocx mallctl mallctlnametomib @@ -120,6 +121,12 @@ void *ptr int flags + + void sdallocx + void *ptr + size_t size + int flags + size_t nallocx size_t size @@ -228,7 +235,8 @@ rallocx, xallocx, sallocx, - dallocx, and + dallocx, + sdallocx, and nallocx functions all have a flags argument that can be used to specify options. The functions only check the options that are contextually @@ -312,6 +320,15 @@ memory referenced by ptr to be made available for future allocations. + The sdallocx function is an + extension of dallocx with a + size parameter to allow the caller to pass in the + allocation size as an optimization. The minimum valid input size is the + original requested size of the allocation, and the maximum valid input + size is the corresponding value returned by + nallocx or + sallocx. + The nallocx function allocates no memory, but it performs the same size computation as the mallocx function, and returns the real diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 166d0523..6ab0ae71 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -488,6 +488,7 @@ void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); +void arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) @@ -1139,9 +1140,7 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) if ((mapbits & CHUNK_MAP_LARGE) == 0) { /* Small allocation. */ if (try_tcache && (tcache = tcache_get(false)) != NULL) { - size_t binind; - - binind = arena_ptr_small_binind_get(ptr, mapbits); + size_t binind = arena_ptr_small_binind_get(ptr, mapbits); tcache_dalloc_small(tcache, ptr, binind); } else arena_dalloc_small(chunk->arena, chunk, ptr, pageind); @@ -1157,6 +1156,34 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) arena_dalloc_large(chunk->arena, chunk, ptr); } } + +JEMALLOC_ALWAYS_INLINE void +arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache) +{ + tcache_t *tcache; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + if (size < PAGE) { + /* Small allocation. */ + if (try_tcache && (tcache = tcache_get(false)) != NULL) { + size_t binind = small_size2bin(size); + tcache_dalloc_small(tcache, ptr, binind); + } else { + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + arena_dalloc_small(chunk->arena, chunk, ptr, pageind); + } + } else { + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + + if (try_tcache && size <= tcache_maxclass && (tcache = + tcache_get(false)) != NULL) { + tcache_dalloc_large(tcache, ptr, size); + } else + arena_dalloc_large(chunk->arena, chunk, ptr); + } +} # endif /* JEMALLOC_ARENA_INLINE_C */ #endif diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 59ae8d55..c0e326d4 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -634,8 +634,10 @@ size_t ivsalloc(const void *ptr, bool demote); size_t u2rz(size_t usize); size_t p2rz(const void *ptr); void idalloct(void *ptr, bool try_tcache); +void isdalloct(void *ptr, size_t size, bool try_tcache); void idalloc(void *ptr); void iqalloc(void *ptr, bool try_tcache); +void isqalloc(void *ptr, size_t size, bool try_tcache); void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); @@ -787,6 +789,20 @@ idalloct(void *ptr, bool try_tcache) huge_dalloc(ptr); } +JEMALLOC_ALWAYS_INLINE void +isdalloct(void *ptr, size_t size, bool try_tcache) +{ + arena_chunk_t *chunk; + + assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) + arena_sdalloc(chunk, ptr, size, try_tcache); + else + huge_dalloc(ptr); +} + JEMALLOC_ALWAYS_INLINE void idalloc(void *ptr) { @@ -804,6 +820,16 @@ iqalloc(void *ptr, bool try_tcache) idalloct(ptr, try_tcache); } +JEMALLOC_ALWAYS_INLINE void +isqalloc(void *ptr, size_t size, bool try_tcache) +{ + + if (config_fill && opt_quarantine) + quarantine(ptr); + else + idalloct(ptr, try_tcache); +} + JEMALLOC_ALWAYS_INLINE void * iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 84f05910..3b990b0e 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -61,6 +61,7 @@ arena_ralloc_no_move arena_redzone_corruption arena_run_regind arena_salloc +arena_sdalloc arena_stats_merge arena_tcache_fill_small arenas @@ -228,7 +229,9 @@ iralloc iralloct iralloct_realign isalloc +isdalloct isthreaded +isqalloc ivsalloc ixalloc jemalloc_postfork_child diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in index b365eb4a..f81adc14 100644 --- a/include/jemalloc/jemalloc_protos.h.in +++ b/include/jemalloc/jemalloc_protos.h.in @@ -25,6 +25,7 @@ JEMALLOC_EXPORT size_t @je_@xallocx(void *ptr, size_t size, size_t extra, JEMALLOC_EXPORT size_t @je_@sallocx(const void *ptr, int flags) JEMALLOC_ATTR(pure); JEMALLOC_EXPORT void @je_@dallocx(void *ptr, int flags); +JEMALLOC_EXPORT void @je_@sdallocx(void *ptr, size_t size, int flags); JEMALLOC_EXPORT size_t @je_@nallocx(size_t size, int flags) JEMALLOC_ATTR(pure); diff --git a/src/jemalloc.c b/src/jemalloc.c index 71e921b5..527782e8 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1223,6 +1223,24 @@ ifree(void *ptr, bool try_tcache) JEMALLOC_VALGRIND_FREE(ptr, rzsize); } +JEMALLOC_INLINE_C void +isfree(void *ptr, size_t usize, bool try_tcache) +{ + UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); + + assert(ptr != NULL); + assert(malloc_initialized || IS_INITIALIZER); + + if (config_prof && opt_prof) + prof_free(ptr, usize); + if (config_stats) + thread_allocated_tsd_get()->deallocated += usize; + if (config_valgrind && in_valgrind) + rzsize = p2rz(ptr); + isqalloc(ptr, usize, try_tcache); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); +} + void * je_realloc(void *ptr, size_t size) { @@ -1820,6 +1838,32 @@ je_dallocx(void *ptr, int flags) ifree(ptr, try_tcache); } +void +je_sdallocx(void *ptr, size_t size, int flags) +{ + bool try_tcache; + + assert(ptr != NULL); + assert(malloc_initialized || IS_INITIALIZER); + assert(size == isalloc(ptr, config_prof)); + + if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) + size = s2u(size); + else + size = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); + + if ((flags & MALLOCX_ARENA_MASK) != 0) { + unsigned arena_ind = MALLOCX_ARENA_GET(flags); + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + try_tcache = (chunk == ptr || chunk->arena != + arenas[arena_ind]); + } else + try_tcache = true; + + UTRACE(ptr, 0, 0); + isfree(ptr, size, try_tcache); +} + size_t je_nallocx(size_t size, int flags) { diff --git a/test/integration/sdallocx.c b/test/integration/sdallocx.c new file mode 100644 index 00000000..b84817d7 --- /dev/null +++ b/test/integration/sdallocx.c @@ -0,0 +1,57 @@ +#include "test/jemalloc_test.h" + +#define MAXALIGN (((size_t)1) << 25) +#define NITER 4 + +TEST_BEGIN(test_basic) +{ + void *ptr = mallocx(64, 0); + sdallocx(ptr, 64, 0); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + size_t nsz, sz, alignment, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + total += nsz; + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + sdallocx(ps[i], sz, + MALLOCX_ALIGN(alignment)); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_basic, + test_alignment_and_size)); +} diff --git a/test/stress/microbench.c b/test/stress/microbench.c index 60c02db3..a8267c39 100644 --- a/test/stress/microbench.c +++ b/test/stress/microbench.c @@ -72,6 +72,17 @@ malloc_dallocx(void) dallocx(p, 0); } +static void +malloc_sdallocx(void) +{ + void *p = malloc(1); + if (p == NULL) { + test_fail("Unexpected malloc() failure"); + return; + } + sdallocx(p, 1, 0); +} + TEST_BEGIN(test_free_vs_dallocx) { @@ -80,6 +91,14 @@ TEST_BEGIN(test_free_vs_dallocx) } TEST_END +TEST_BEGIN(test_dallocx_vs_sdallocx) +{ + + compare_funcs(10*1000*1000, 100*1000*1000, "dallocx", malloc_dallocx, + "sdallocx", malloc_sdallocx); +} +TEST_END + static void malloc_mus_free(void) { @@ -135,6 +154,7 @@ main(void) return (test( test_malloc_vs_mallocx, test_free_vs_dallocx, + test_dallocx_vs_sdallocx, test_mus_vs_sallocx, test_sallocx_vs_nallocx)); } From a62812eacca8ac3ce81f27c9480b44b2a97ff66c Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Mon, 8 Sep 2014 21:43:21 -0400 Subject: [PATCH 095/352] fix isqalloct (should call isdalloct) --- include/jemalloc/internal/jemalloc_internal.h.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index c0e326d4..81d46fc3 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -827,7 +827,7 @@ isqalloc(void *ptr, size_t size, bool try_tcache) if (config_fill && opt_quarantine) quarantine(ptr); else - idalloct(ptr, try_tcache); + isdalloct(ptr, size, try_tcache); } JEMALLOC_ALWAYS_INLINE void * From d95e704feadd44cc6d9eb8695b9cff7ac6d4c88f Mon Sep 17 00:00:00 2001 From: Bert Maher Date: Fri, 5 Sep 2014 14:10:37 -0700 Subject: [PATCH 096/352] Support threaded heap profiles in pprof - Add a --thread N option to select profile for thread N (otherwise, all threads will be printed) - The $profile map now has a {threads} element that is a map from thread id to a profile that has the same format as the {profile} element - Refactor ReadHeapProfile into smaller components and use them to implement ReadThreadedHeapProfile --- bin/pprof | 405 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 265 insertions(+), 140 deletions(-) diff --git a/bin/pprof b/bin/pprof index 328138cd..52da6004 100755 --- a/bin/pprof +++ b/bin/pprof @@ -2,11 +2,11 @@ # Copyright (c) 1998-2007, Google Inc. # All rights reserved. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: -# +# # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above @@ -16,7 +16,7 @@ # * Neither the name of Google Inc. nor the names of its # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR @@ -223,6 +223,7 @@ Call-graph Options: --edgefraction= Hide edges below *total [default=.001] --maxdegree= Max incoming/outgoing edges per node [default=8] --focus= Focus on nodes matching + --thread= Show profile for thread --ignore= Ignore nodes matching --scale= Set GV scaling [default=0] --heapcheck Make nodes with non-0 object counts @@ -332,6 +333,7 @@ sub Init() { $main::opt_edgefraction = 0.001; $main::opt_maxdegree = 8; $main::opt_focus = ''; + $main::opt_thread = undef; $main::opt_ignore = ''; $main::opt_scale = 0; $main::opt_heapcheck = 0; @@ -402,6 +404,7 @@ sub Init() { "edgefraction=f" => \$main::opt_edgefraction, "maxdegree=i" => \$main::opt_maxdegree, "focus=s" => \$main::opt_focus, + "thread=i" => \$main::opt_thread, "ignore=s" => \$main::opt_ignore, "scale=i" => \$main::opt_scale, "heapcheck" => \$main::opt_heapcheck, @@ -562,6 +565,86 @@ sub Init() { } } +sub FilterAndPrint { + my ($profile, $symbols, $libs, $thread) = @_; + + # Get total data in profile + my $total = TotalProfile($profile); + + # Remove uniniteresting stack items + $profile = RemoveUninterestingFrames($symbols, $profile); + + # Focus? + if ($main::opt_focus ne '') { + $profile = FocusProfile($symbols, $profile, $main::opt_focus); + } + + # Ignore? + if ($main::opt_ignore ne '') { + $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); + } + + my $calls = ExtractCalls($symbols, $profile); + + # Reduce profiles to required output granularity, and also clean + # each stack trace so a given entry exists at most once. + my $reduced = ReduceProfile($symbols, $profile); + + # Get derived profiles + my $flat = FlatProfile($reduced); + my $cumulative = CumulativeProfile($reduced); + + # Print + if (!$main::opt_interactive) { + if ($main::opt_disasm) { + PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm); + } elsif ($main::opt_list) { + PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); + } elsif ($main::opt_text) { + # Make sure the output is empty when have nothing to report + # (only matters when --heapcheck is given but we must be + # compatible with old branches that did not pass --heapcheck always): + if ($total != 0) { + printf("Total%s: %s %s\n", + (defined($thread) ? " (t$thread)" : ""), + Unparse($total), Units()); + } + PrintText($symbols, $flat, $cumulative, -1); + } elsif ($main::opt_raw) { + PrintSymbolizedProfile($symbols, $profile, $main::prog); + } elsif ($main::opt_callgrind) { + PrintCallgrind($calls); + } else { + if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { + if ($main::opt_gv) { + RunGV(TempName($main::next_tmpfile, "ps"), ""); + } elsif ($main::opt_evince) { + RunEvince(TempName($main::next_tmpfile, "pdf"), ""); + } elsif ($main::opt_web) { + my $tmp = TempName($main::next_tmpfile, "svg"); + RunWeb($tmp); + # The command we run might hand the file name off + # to an already running browser instance and then exit. + # Normally, we'd remove $tmp on exit (right now), + # but fork a child to remove $tmp a little later, so that the + # browser has time to load it first. + delete $main::tempnames{$tmp}; + if (fork() == 0) { + sleep 5; + unlink($tmp); + exit(0); + } + } + } else { + cleanup(); + exit(1); + } + } + } else { + InteractiveMode($profile, $symbols, $libs, $total); + } +} + sub Main() { Init(); $main::collected_profile = undef; @@ -605,9 +688,6 @@ sub Main() { $symbol_map = MergeSymbols($symbol_map, $base->{symbols}); } - # Get total data in profile - my $total = TotalProfile($profile); - # Collect symbols my $symbols; if ($main::use_symbolized_profile) { @@ -622,75 +702,16 @@ sub Main() { $symbols = ExtractSymbols($libs, $pcs); } - # Remove uniniteresting stack items - $profile = RemoveUninterestingFrames($symbols, $profile); - - # Focus? - if ($main::opt_focus ne '') { - $profile = FocusProfile($symbols, $profile, $main::opt_focus); + if (!defined($main::opt_thread)) { + FilterAndPrint($profile, $symbols, $libs); } - - # Ignore? - if ($main::opt_ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); - } - - my $calls = ExtractCalls($symbols, $profile); - - # Reduce profiles to required output granularity, and also clean - # each stack trace so a given entry exists at most once. - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - # Print - if (!$main::opt_interactive) { - if ($main::opt_disasm) { - PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm); - } elsif ($main::opt_list) { - PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); - } elsif ($main::opt_text) { - # Make sure the output is empty when have nothing to report - # (only matters when --heapcheck is given but we must be - # compatible with old branches that did not pass --heapcheck always): - if ($total != 0) { - printf("Total: %s %s\n", Unparse($total), Units()); - } - PrintText($symbols, $flat, $cumulative, -1); - } elsif ($main::opt_raw) { - PrintSymbolizedProfile($symbols, $profile, $main::prog); - } elsif ($main::opt_callgrind) { - PrintCallgrind($calls); - } else { - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), ""); - } elsif ($main::opt_evince) { - RunEvince(TempName($main::next_tmpfile, "pdf"), ""); - } elsif ($main::opt_web) { - my $tmp = TempName($main::next_tmpfile, "svg"); - RunWeb($tmp); - # The command we run might hand the file name off - # to an already running browser instance and then exit. - # Normally, we'd remove $tmp on exit (right now), - # but fork a child to remove $tmp a little later, so that the - # browser has time to load it first. - delete $main::tempnames{$tmp}; - if (fork() == 0) { - sleep 5; - unlink($tmp); - exit(0); - } - } - } else { - cleanup(); - exit(1); + if (defined($data->{threads})) { + foreach my $thread (sort { $a <=> $b } keys(%{$data->{threads}})) { + if (!defined($main::opt_thread) || $main::opt_thread == $thread) { + my $thread_profile = $data->{threads}{$thread}; + FilterAndPrint($thread_profile, $symbols, $libs, $thread); } } - } else { - InteractiveMode($profile, $symbols, $libs, $total); } cleanup(); @@ -1683,23 +1704,23 @@ sub PrintSource { HtmlPrintNumber($c2), UnparseAddress($offset, $e->[0]), CleanDisassembly($e->[3])); - + # Append the most specific source line associated with this instruction if (length($dis) < 80) { $dis .= (' ' x (80 - length($dis))) }; $dis = HtmlEscape($dis); my $f = $e->[5]; my $l = $e->[6]; if ($f ne $last_dis_filename) { - $dis .= sprintf("%s:%d", + $dis .= sprintf("%s:%d", HtmlEscape(CleanFileName($f)), $l); } elsif ($l ne $last_dis_linenum) { # De-emphasize the unchanged file name portion $dis .= sprintf("%s" . - ":%d", + ":%d", HtmlEscape(CleanFileName($f)), $l); } else { # De-emphasize the entire location - $dis .= sprintf("%s:%d", + $dis .= sprintf("%s:%d", HtmlEscape(CleanFileName($f)), $l); } $last_dis_filename = $f; @@ -1788,8 +1809,8 @@ sub PrintSource { if (defined($dis) && $dis ne '') { $asm = "" . $dis . ""; } - my $source_class = (($n1 + $n2 > 0) - ? "livesrc" + my $source_class = (($n1 + $n2 > 0) + ? "livesrc" : (($asm ne "") ? "deadsrc" : "nop")); printf $output ( "%5d " . @@ -3689,6 +3710,7 @@ sub IsSymbolizedProfileFile { # $result->{version} Version number of profile file # $result->{period} Sampling period (in microseconds) # $result->{profile} Profile object +# $result->{threads} Map of thread IDs to profile objects # $result->{map} Memory map info from profile # $result->{pcs} Hash of all PC values seen, key is hex address sub ReadProfile { @@ -3737,6 +3759,9 @@ sub ReadProfile { } elsif ($header =~ m/^heap profile:/) { $main::profile_type = 'heap'; $result = ReadHeapProfile($prog, *PROFILE, $header); + } elsif ($header =~ m/^heap/) { + $main::profile_type = 'heap'; + $result = ReadThreadedHeapProfile($prog, $fname, $header); } elsif ($header =~ m/^--- *$contention_marker/o) { $main::profile_type = 'contention'; $result = ReadSynchProfile($prog, *PROFILE); @@ -3879,11 +3904,7 @@ sub ReadCPUProfile { return $r; } -sub ReadHeapProfile { - my $prog = shift; - local *PROFILE = shift; - my $header = shift; - +sub HeapProfileIndex { my $index = 1; if ($main::opt_inuse_space) { $index = 1; @@ -3894,6 +3915,84 @@ sub ReadHeapProfile { } elsif ($main::opt_alloc_objects) { $index = 2; } + return $index; +} + +sub ReadMappedLibraries { + my $fh = shift; + my $map = ""; + # Read the /proc/self/maps data + while (<$fh>) { + s/\r//g; # turn windows-looking lines into unix-looking lines + $map .= $_; + } + return $map; +} + +sub ReadMemoryMap { + my $fh = shift; + my $map = ""; + # Read /proc/self/maps data as formatted by DumpAddressMap() + my $buildvar = ""; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + # Parse "build=" specification if supplied + if (m/^\s*build=(.*)\n/) { + $buildvar = $1; + } + + # Expand "$build" variable if available + $_ =~ s/\$build\b/$buildvar/g; + + $map .= $_; + } + return $map; +} + +sub AdjustSamples { + my ($sample_adjustment, $sampling_algorithm, $n1, $s1, $n2, $s2) = @_; + if ($sample_adjustment) { + if ($sampling_algorithm == 2) { + # Remote-heap version 2 + # The sampling frequency is the rate of a Poisson process. + # This means that the probability of sampling an allocation of + # size X with sampling rate Y is 1 - exp(-X/Y) + if ($n1 != 0) { + my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n1 *= $scale_factor; + $s1 *= $scale_factor; + } + if ($n2 != 0) { + my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n2 *= $scale_factor; + $s2 *= $scale_factor; + } + } else { + # Remote-heap version 1 + my $ratio; + $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + if ($ratio < 1) { + $n1 /= $ratio; + $s1 /= $ratio; + } + $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + if ($ratio < 1) { + $n2 /= $ratio; + $s2 /= $ratio; + } + } + } + return ($n1, $s1, $n2, $s2); +} + +sub ReadHeapProfile { + my $prog = shift; + local *PROFILE = shift; + my $header = shift; + + my $index = HeapProfileIndex(); # Find the type of this profile. The header line looks like: # heap profile: 1246: 8800744 [ 1246: 8800744] @ /266053 @@ -3983,29 +4082,12 @@ sub ReadHeapProfile { while () { s/\r//g; # turn windows-looking lines into unix-looking lines if (/^MAPPED_LIBRARIES:/) { - # Read the /proc/self/maps data - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $map .= $_; - } + $map .= ReadMappedLibraries(*PROFILE); last; } if (/^--- Memory map:/) { - # Read /proc/self/maps data as formatted by DumpAddressMap() - my $buildvar = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Parse "build=" specification if supplied - if (m/^\s*build=(.*)\n/) { - $buildvar = $1; - } - - # Expand "$build" variable if available - $_ =~ s/\$build\b/$buildvar/g; - - $map .= $_; - } + $map .= ReadMemoryMap(*PROFILE); last; } @@ -4016,42 +4098,8 @@ sub ReadHeapProfile { if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) { my $stack = $5; my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); - - if ($sample_adjustment) { - if ($sampling_algorithm == 2) { - # Remote-heap version 2 - # The sampling frequency is the rate of a Poisson process. - # This means that the probability of sampling an allocation of - # size X with sampling rate Y is 1 - exp(-X/Y) - if ($n1 != 0) { - my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n1 *= $scale_factor; - $s1 *= $scale_factor; - } - if ($n2 != 0) { - my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n2 *= $scale_factor; - $s2 *= $scale_factor; - } - } else { - # Remote-heap version 1 - my $ratio; - $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - if ($ratio < 1) { - $n1 /= $ratio; - $s1 /= $ratio; - } - $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - if ($ratio < 1) { - $n2 /= $ratio; - $s2 /= $ratio; - } - } - } - - my @counts = ($n1, $s1, $n2, $s2); + my @counts = AdjustSamples($sample_adjustment, $sampling_algorithm, + $n1, $s1, $n2, $s2); AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); } } @@ -4065,6 +4113,83 @@ sub ReadHeapProfile { return $r; } +sub ReadThreadedHeapProfile { + my ($prog, $fname, $header) = @_; + + my $index = HeapProfileIndex(); + my $sampling_algorithm = 0; + my $sample_adjustment = 0; + chomp($header); + my $type = "unknown"; + # Assuming a very specific type of header for now. + if ($header =~ m"^heap_v2/(\d+)") { + $type = "_v2"; + $sampling_algorithm = 2; + $sample_adjustment = int($1); + } + if ($type ne "_v2" || !defined($sample_adjustment)) { + die "Threaded heap profiles require v2 sampling with a sample rate\n"; + } + + my $profile = {}; + my $thread_profiles = {}; + my $pcs = {}; + my $map = ""; + my $stack = ""; + + while () { + s/\r//g; + if (/^MAPPED_LIBRARIES:/) { + $map .= ReadMappedLibraries(*PROFILE); + last; + } + + if (/^--- Memory map:/) { + $map .= ReadMemoryMap(*PROFILE); + last; + } + + # Read entry of the form: + # @ a1 a2 ... an + # t*: : [: ] + # t1: : [: ] + # ... + # tn: : [: ] + s/^\s*//; + s/\s*$//; + if (m/^@\s+(.*)$/) { + $stack = $1; + } elsif (m/^\s*(t(\*|\d+)):\s+(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]$/) { + if ($stack eq "") { + # Still in the header, so this is just a per-thread summary. + next; + } + my $thread = $2; + my ($n1, $s1, $n2, $s2) = ($3, $4, $5, $6); + my @counts = AdjustSamples($sample_adjustment, $sampling_algorithm, + $n1, $s2, $n2, $s2); + if ($thread eq "*") { + AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); + } else { + if (!exists($thread_profiles->{$thread})) { + $thread_profiles->{$thread} = {}; + } + AddEntries($thread_profiles->{$thread}, $pcs, + FixCallerAddresses($stack), $counts[$index]); + } + } + } + + my $r = {}; + $r->{version} = "heap"; + $r->{period} = 1; + $r->{profile} = $profile; + $r->{threads} = $thread_profiles; + $r->{libs} = ParseLibraries($prog, $map, $pcs); + $r->{pcs} = $pcs; + return $r; +} + sub ReadSynchProfile { my $prog = shift; local *PROFILE = shift; @@ -4756,7 +4881,7 @@ sub MapToSymbols { } } } - + # Prepend to accumulated symbols for pcstr # (so that caller comes before callee) my $sym = $symbols->{$pcstr}; @@ -4950,7 +5075,7 @@ sub ConfigureTool { my $dirname = $`; # this is everything up to and including the last slash if (-x "$dirname$tool") { $path = "$dirname$tool"; - } else { + } else { $path = $tool; } } From a2260c95cd717c06c28b61d40b2157254d594219 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 9 Sep 2014 10:29:26 -0700 Subject: [PATCH 097/352] Fix sdallocx() assertion. Refactor sdallocx() and nallocx() to share inallocx(), and fix an sdallocx() assertion to check usize rather than size. --- src/jemalloc.c | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 527782e8..3f29a857 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1838,19 +1838,29 @@ je_dallocx(void *ptr, int flags) ifree(ptr, try_tcache); } +JEMALLOC_ALWAYS_INLINE_C size_t +inallocx(size_t size, int flags) +{ + size_t usize; + + if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) + usize = s2u(size); + else + usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); + assert(usize != 0); + return (usize); +} + void je_sdallocx(void *ptr, size_t size, int flags) { bool try_tcache; + size_t usize; assert(ptr != NULL); assert(malloc_initialized || IS_INITIALIZER); - assert(size == isalloc(ptr, config_prof)); - - if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) - size = s2u(size); - else - size = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); + usize = inallocx(size, flags); + assert(usize == isalloc(ptr, config_prof)); if ((flags & MALLOCX_ARENA_MASK) != 0) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); @@ -1861,27 +1871,19 @@ je_sdallocx(void *ptr, size_t size, int flags) try_tcache = true; UTRACE(ptr, 0, 0); - isfree(ptr, size, try_tcache); + isfree(ptr, usize, try_tcache); } size_t je_nallocx(size_t size, int flags) { - size_t usize; assert(size != 0); if (malloc_init()) return (0); - if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) - usize = s2u(size); - else { - size_t alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); - usize = sa2u(size, alignment); - } - assert(usize != 0); - return (usize); + return (inallocx(size, flags)); } int From 7c17e1670d7294db4b3c483ad7173dd056b42268 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 9 Sep 2014 15:27:52 -0700 Subject: [PATCH 098/352] Fix threaded heap profile bug in pprof. Fix ReadThreadedHeapProfile to pass the correct parameters to AdjustSamples. --- bin/pprof | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/pprof b/bin/pprof index 52da6004..87313f43 100755 --- a/bin/pprof +++ b/bin/pprof @@ -4167,7 +4167,7 @@ sub ReadThreadedHeapProfile { my $thread = $2; my ($n1, $s1, $n2, $s2) = ($3, $4, $5, $6); my @counts = AdjustSamples($sample_adjustment, $sampling_algorithm, - $n1, $s2, $n2, $s2); + $n1, $s1, $n2, $s2); if ($thread eq "*") { AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); } else { From 6fd53da030b5e9161a49d6010a8b38499ca2a124 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 9 Sep 2014 12:45:53 -0700 Subject: [PATCH 099/352] Fix prof_tdata_get()-related regressions. Fix prof_tdata_get() to avoid dereferencing an invalid tdata pointer (when it's PROF_TDATA_STATE_{REINCARNATED,PURGATORY}). Fix prof_tdata_get() callers to check for invalid results besides NULL (PROF_TDATA_STATE_{REINCARNATED,PURGATORY}). These regressions were caused by 602c8e0971160e4b85b08b16cf8a2375aa24bc04 (Implement per thread heap profiling.), which did not make it into any releases prior to these fixes. --- include/jemalloc/internal/prof.h | 11 ++++---- src/prof.c | 45 ++++++++++++++------------------ 2 files changed, 26 insertions(+), 30 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 104bfade..a9903280 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -308,12 +308,13 @@ prof_tdata_get(bool create) tdata = *prof_tdata_tsd_get(); if (create) { - if (tdata == NULL) - tdata = prof_tdata_init(); - else if (tdata->state == prof_tdata_state_expired) + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { + if (tdata == NULL) + tdata = prof_tdata_init(); + } else if (tdata->state == prof_tdata_state_expired) tdata = prof_tdata_reinit(tdata); - assert(tdata == NULL || tdata->state == - prof_tdata_state_attached); + assert((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX || + tdata->state == prof_tdata_state_attached); } return (tdata); diff --git a/src/prof.c b/src/prof.c index 044acd8b..941e53be 100644 --- a/src/prof.c +++ b/src/prof.c @@ -487,9 +487,8 @@ prof_gctx_create(prof_bt_t *bt) } static void -prof_gctx_maybe_destroy(prof_gctx_t *gctx) +prof_gctx_maybe_destroy(prof_gctx_t *gctx, prof_tdata_t *tdata) { - prof_tdata_t *tdata; cassert(config_prof); @@ -500,8 +499,6 @@ prof_gctx_maybe_destroy(prof_gctx_t *gctx) * avoid a race between the main body of prof_tctx_destroy() and entry * into this function. */ - tdata = prof_tdata_get(false); - assert((uintptr_t)tdata > (uintptr_t)PROF_TDATA_STATE_MAX); prof_enter(tdata); malloc_mutex_lock(gctx->lock); if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { @@ -552,8 +549,9 @@ prof_gctx_should_destroy(prof_gctx_t *gctx) static void prof_tctx_destroy(prof_tctx_t *tctx) { + prof_tdata_t *tdata = tctx->tdata; prof_gctx_t *gctx = tctx->gctx; - bool destroy_gctx; + bool destroy_tdata, destroy_gctx; assert(tctx->cnts.curobjs == 0); assert(tctx->cnts.curbytes == 0); @@ -561,16 +559,9 @@ prof_tctx_destroy(prof_tctx_t *tctx) assert(tctx->cnts.accumobjs == 0); assert(tctx->cnts.accumbytes == 0); - { - prof_tdata_t *tdata = tctx->tdata; - bool tdata_destroy; - - ckh_remove(&tdata->bt2tctx, &gctx->bt, NULL, NULL); - tdata_destroy = prof_tdata_should_destroy(tdata); - malloc_mutex_unlock(tdata->lock); - if (tdata_destroy) - prof_tdata_destroy(tdata); - } + ckh_remove(&tdata->bt2tctx, &gctx->bt, NULL, NULL); + destroy_tdata = prof_tdata_should_destroy(tdata); + malloc_mutex_unlock(tdata->lock); malloc_mutex_lock(gctx->lock); tctx_tree_remove(&gctx->tctxs, tctx); @@ -594,7 +585,10 @@ prof_tctx_destroy(prof_tctx_t *tctx) destroy_gctx = false; malloc_mutex_unlock(gctx->lock); if (destroy_gctx) - prof_gctx_maybe_destroy(gctx); + prof_gctx_maybe_destroy(gctx, tdata); + + if (destroy_tdata) + prof_tdata_destroy(tdata); idalloc(tctx); } @@ -683,7 +677,7 @@ prof_lookup(prof_bt_t *bt) ret.v = imalloc(sizeof(prof_tctx_t)); if (ret.p == NULL) { if (new_gctx) - prof_gctx_maybe_destroy(gctx); + prof_gctx_maybe_destroy(gctx, tdata); return (NULL); } ret.p->tdata = tdata; @@ -695,7 +689,7 @@ prof_lookup(prof_bt_t *bt) malloc_mutex_unlock(tdata->lock); if (error) { if (new_gctx) - prof_gctx_maybe_destroy(gctx); + prof_gctx_maybe_destroy(gctx, tdata); idalloc(ret.v); return (NULL); } @@ -1019,6 +1013,7 @@ prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) static prof_gctx_t * prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) { + prof_tdata_t *tdata = (prof_tdata_t *)arg; prof_tctx_t *next; bool destroy_gctx; @@ -1032,7 +1027,7 @@ prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) destroy_gctx = prof_gctx_should_destroy(gctx); malloc_mutex_unlock(gctx->lock); if (destroy_gctx) - prof_gctx_maybe_destroy(gctx); + prof_gctx_maybe_destroy(gctx, tdata); return (NULL); } @@ -1310,7 +1305,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) if (prof_dump_close(propagate_err)) goto label_open_close_error; - gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, NULL); + gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata); malloc_mutex_unlock(&prof_dump_mtx); if (leakcheck) @@ -1320,7 +1315,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) label_write_error: prof_dump_close(propagate_err); label_open_close_error: - gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, NULL); + gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata); malloc_mutex_unlock(&prof_dump_mtx); return (true); } @@ -1643,7 +1638,7 @@ const char * prof_thread_name_get(void) { prof_tdata_t *tdata = prof_tdata_get(true); - if (tdata == NULL) + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (NULL); return (tdata->thread_name); } @@ -1656,7 +1651,7 @@ prof_thread_name_set(const char *thread_name) char *s; tdata = prof_tdata_get(true); - if (tdata == NULL) + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (true); size = strlen(thread_name) + 1; @@ -1675,7 +1670,7 @@ bool prof_thread_active_get(void) { prof_tdata_t *tdata = prof_tdata_get(true); - if (tdata == NULL) + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (false); return (tdata->active); } @@ -1686,7 +1681,7 @@ prof_thread_active_set(bool active) prof_tdata_t *tdata; tdata = prof_tdata_get(true); - if (tdata == NULL) + if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (true); tdata->active = active; return (false); From 6e73dc194ee9682d3eacaf725a989f04629718f7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 9 Sep 2014 19:37:26 -0700 Subject: [PATCH 100/352] Fix a profile sampling race. Fix a profile sampling race that was due to preparing to sample, yet doing nothing to assure that the context remains valid until the stats are updated. These regressions were caused by 602c8e0971160e4b85b08b16cf8a2375aa24bc04 (Implement per thread heap profiling.), which did not make it into any releases prior to these fixes. --- include/jemalloc/internal/private_symbols.txt | 1 + include/jemalloc/internal/prof.h | 37 +++--- src/jemalloc.c | 109 +++++++++--------- src/prof.c | 35 ++++++ 4 files changed, 109 insertions(+), 73 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 3b990b0e..b8990177 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -292,6 +292,7 @@ p2rz pages_purge pow2_ceil prof_alloc_prep +prof_alloc_rollback prof_backtrace prof_boot0 prof_boot1 diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index a9903280..920ec63f 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -97,6 +97,12 @@ struct prof_tctx_s { /* Linkage into gctx's tctxs. */ rb_node(prof_tctx_t) tctx_link; + /* + * True during prof_alloc_prep()..prof_malloc_sample_object(), prevents + * sample vs destroy race. + */ + bool prepared; + /* Current dump-related state, protected by gctx->lock. */ prof_tctx_state_t state; @@ -242,6 +248,7 @@ extern uint64_t prof_interval; */ extern size_t lg_prof_sample; +void prof_alloc_rollback(prof_tctx_t *tctx, bool updated); void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_free_sampled_object(size_t usize, prof_tctx_t *tctx); @@ -282,14 +289,14 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) prof_tdata_t *prof_tdata_get(bool create); bool prof_sample_accum_update(size_t usize, bool commit, prof_tdata_t **tdata_out); -prof_tctx_t *prof_alloc_prep(size_t usize); +prof_tctx_t *prof_alloc_prep(size_t usize, bool update); prof_tctx_t *prof_tctx_get(const void *ptr); void prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, - size_t old_usize, prof_tctx_t *old_tctx); + bool updated, size_t old_usize, prof_tctx_t *old_tctx); void prof_free(const void *ptr, size_t usize); #endif @@ -356,7 +363,7 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx) } JEMALLOC_INLINE bool -prof_sample_accum_update(size_t usize, bool commit, prof_tdata_t **tdata_out) +prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out) { prof_tdata_t *tdata; @@ -373,19 +380,19 @@ prof_sample_accum_update(size_t usize, bool commit, prof_tdata_t **tdata_out) return (true); if (tdata->bytes_until_sample >= usize) { - if (commit) + if (update) tdata->bytes_until_sample -= usize; return (true); } else { /* Compute new sample threshold. */ - if (commit) + if (update) prof_sample_threshold_update(tdata); return (tdata->active == false); } } JEMALLOC_INLINE prof_tctx_t * -prof_alloc_prep(size_t usize) +prof_alloc_prep(size_t usize, bool update) { prof_tctx_t *ret; prof_tdata_t *tdata; @@ -393,7 +400,7 @@ prof_alloc_prep(size_t usize) assert(usize == s2u(usize)); - if (!opt_prof_active || prof_sample_accum_update(usize, false, &tdata)) + if (!opt_prof_active || prof_sample_accum_update(usize, update, &tdata)) ret = (prof_tctx_t *)(uintptr_t)1U; else { bt_init(&bt, tdata->vec); @@ -412,16 +419,6 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) assert(ptr != NULL); assert(usize == isalloc(ptr, true)); - if (prof_sample_accum_update(usize, true, NULL)) { - /* - * Don't sample. For malloc()-like allocation, it is always - * possible to tell in advance how large an object's usable size - * will be, so there should never be a difference between the - * usize passed to PROF_ALLOC_PREP() and prof_malloc(). - */ - assert((uintptr_t)tctx == (uintptr_t)1U); - } - if ((uintptr_t)tctx > (uintptr_t)1U) prof_malloc_sample_object(ptr, usize, tctx); else @@ -429,14 +426,14 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) } JEMALLOC_INLINE void -prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, size_t old_usize, - prof_tctx_t *old_tctx) +prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated, + size_t old_usize, prof_tctx_t *old_tctx) { cassert(config_prof); assert(ptr != NULL || (uintptr_t)tctx <= (uintptr_t)1U); - if (ptr != NULL) { + if (!updated && ptr != NULL) { assert(usize == isalloc(ptr, true)); if (prof_sample_accum_update(usize, true, NULL)) { /* diff --git a/src/jemalloc.c b/src/jemalloc.c index 3f29a857..1d4d1a8a 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -886,13 +886,15 @@ imalloc_prof(size_t usize) void *p; prof_tctx_t *tctx; - tctx = prof_alloc_prep(usize); + tctx = prof_alloc_prep(usize, true); if ((uintptr_t)tctx != (uintptr_t)1U) p = imalloc_prof_sample(usize, tctx); else p = imalloc(usize); - if (p == NULL) + if (p == NULL) { + prof_alloc_rollback(tctx, true); return (NULL); + } prof_malloc(p, usize, tctx); return (p); @@ -962,16 +964,20 @@ imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx) } JEMALLOC_ALWAYS_INLINE_C void * -imemalign_prof(size_t alignment, size_t usize, prof_tctx_t *tctx) +imemalign_prof(size_t alignment, size_t usize) { void *p; + prof_tctx_t *tctx; + tctx = prof_alloc_prep(usize, true); if ((uintptr_t)tctx != (uintptr_t)1U) p = imemalign_prof_sample(alignment, usize, tctx); else p = ipalloc(usize, alignment, false); - if (p == NULL) + if (p == NULL) { + prof_alloc_rollback(tctx, true); return (NULL); + } prof_malloc(p, usize, tctx); return (p); @@ -1013,12 +1019,9 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) goto label_oom; } - if (config_prof && opt_prof) { - prof_tctx_t *tctx; - - tctx = prof_alloc_prep(usize); - result = imemalign_prof(alignment, usize, tctx); - } else + if (config_prof && opt_prof) + result = imemalign_prof(alignment, usize); + else result = ipalloc(usize, alignment, false); if (result == NULL) goto label_oom; @@ -1087,16 +1090,20 @@ icalloc_prof_sample(size_t usize, prof_tctx_t *tctx) } JEMALLOC_ALWAYS_INLINE_C void * -icalloc_prof(size_t usize, prof_tctx_t *tctx) +icalloc_prof(size_t usize) { void *p; + prof_tctx_t *tctx; + tctx = prof_alloc_prep(usize, true); if ((uintptr_t)tctx != (uintptr_t)1U) p = icalloc_prof_sample(usize, tctx); else p = icalloc(usize); - if (p == NULL) + if (p == NULL) { + prof_alloc_rollback(tctx, true); return (NULL); + } prof_malloc(p, usize, tctx); return (p); @@ -1136,11 +1143,8 @@ je_calloc(size_t num, size_t size) } if (config_prof && opt_prof) { - prof_tctx_t *tctx; - usize = s2u(num_size); - tctx = prof_alloc_prep(usize); - ret = icalloc_prof(usize, tctx); + ret = icalloc_prof(usize); } else { if (config_stats || (config_valgrind && in_valgrind)) usize = s2u(num_size); @@ -1184,19 +1188,20 @@ irealloc_prof_sample(void *oldptr, size_t usize, prof_tctx_t *tctx) } JEMALLOC_ALWAYS_INLINE_C void * -irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_tctx_t *tctx) +irealloc_prof(void *oldptr, size_t old_usize, size_t usize) { void *p; - prof_tctx_t *old_tctx; + prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(oldptr); + tctx = prof_alloc_prep(usize, true); if ((uintptr_t)tctx != (uintptr_t)1U) p = irealloc_prof_sample(oldptr, usize, tctx); else p = iralloc(oldptr, usize, 0, false); if (p == NULL) return (NULL); - prof_realloc(p, usize, tctx, old_usize, old_tctx); + prof_realloc(p, usize, tctx, true, old_usize, old_tctx); return (p); } @@ -1270,11 +1275,8 @@ je_realloc(void *ptr, size_t size) old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); if (config_prof && opt_prof) { - prof_tctx_t *tctx; - usize = s2u(size); - tctx = prof_alloc_prep(usize); - ret = irealloc_prof(ptr, old_usize, usize, tctx); + ret = irealloc_prof(ptr, old_usize, usize); } else { if (config_stats || (config_valgrind && in_valgrind)) usize = s2u(size); @@ -1477,7 +1479,7 @@ imallocx_prof(size_t size, int flags, size_t *usize) imallocx_flags_decode(size, flags, usize, &alignment, &zero, &try_tcache, &arena); - tctx = prof_alloc_prep(*usize); + tctx = prof_alloc_prep(*usize, true); if ((uintptr_t)tctx == (uintptr_t)1U) { p = imallocx_maybe_flags(size, flags, *usize, alignment, zero, try_tcache, arena); @@ -1486,8 +1488,10 @@ imallocx_prof(size_t size, int flags, size_t *usize) try_tcache, arena); } else p = NULL; - if (p == NULL) + if (p == NULL) { + prof_alloc_rollback(tctx, true); return (NULL); + } prof_malloc(p, *usize, tctx); return (p); @@ -1572,21 +1576,24 @@ irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, JEMALLOC_ALWAYS_INLINE_C void * irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, - arena_t *arena, prof_tctx_t *tctx) + arena_t *arena) { void *p; - prof_tctx_t *old_tctx; + prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(oldptr); - if ((uintptr_t)tctx != (uintptr_t)1U) + tctx = prof_alloc_prep(*usize, true); + if ((uintptr_t)tctx != (uintptr_t)1U) { p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx); - else { + } else { p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); } - if (p == NULL) + if (p == NULL) { + prof_alloc_rollback(tctx, true); return (NULL); + } if (p == oldptr && alignment != 0) { /* @@ -1599,7 +1606,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, */ *usize = isalloc(p, config_prof); } - prof_realloc(p, *usize, tctx, old_usize, old_tctx); + prof_realloc(p, *usize, tctx, true, old_usize, old_tctx); return (p); } @@ -1641,13 +1648,10 @@ je_rallocx(void *ptr, size_t size, int flags) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { - prof_tctx_t *tctx; - usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); assert(usize != 0); - tctx = prof_alloc_prep(usize); p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, - try_tcache_alloc, try_tcache_dalloc, arena, tctx); + try_tcache_alloc, try_tcache_dalloc, arena); if (p == NULL) goto label_oom; } else { @@ -1720,13 +1724,21 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, JEMALLOC_ALWAYS_INLINE_C size_t ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, - size_t alignment, size_t max_usize, bool zero, arena_t *arena, - prof_tctx_t *tctx) + size_t alignment, bool zero, arena_t *arena) { - size_t usize; - prof_tctx_t *old_tctx; + size_t max_usize, usize; + prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(ptr); + /* + * usize isn't knowable before ixalloc() returns when extra is non-zero. + * Therefore, compute its maximum possible value and use that in + * prof_alloc_prep() to decide whether to capture a backtrace. + * prof_realloc() will use the actual usize to decide whether to sample. + */ + max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, + alignment); + tctx = prof_alloc_prep(max_usize, false); if ((uintptr_t)tctx != (uintptr_t)1U) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, alignment, zero, max_usize, arena, tctx); @@ -1734,9 +1746,11 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero, arena); } - if (usize == old_usize) + if (usize == old_usize) { + prof_alloc_rollback(tctx, false); return (usize); - prof_realloc(ptr, usize, tctx, old_usize, old_tctx); + } + prof_realloc(ptr, usize, tctx, false, old_usize, old_tctx); return (usize); } @@ -1767,19 +1781,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { - prof_tctx_t *tctx; - /* - * usize isn't knowable before ixalloc() returns when extra is - * non-zero. Therefore, compute its maximum possible value and - * use that in prof_alloc_prep() to decide whether to capture a - * backtrace. prof_realloc() will use the actual usize to - * decide whether to sample. - */ - size_t max_usize = (alignment == 0) ? s2u(size+extra) : - sa2u(size+extra, alignment); - tctx = prof_alloc_prep(max_usize); usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, - max_usize, zero, arena, tctx); + zero, arena); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero, arena); diff --git a/src/prof.c b/src/prof.c index 941e53be..9495afc4 100644 --- a/src/prof.c +++ b/src/prof.c @@ -149,6 +149,35 @@ rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, /******************************************************************************/ +void +prof_alloc_rollback(prof_tctx_t *tctx, bool updated) +{ + prof_tdata_t *tdata; + + cassert(config_prof); + + if (updated) { + /* + * Compute a new sample threshold. This isn't very important in + * practice, because this function is rarely executed, so the + * potential for sample bias is minimal except in contrived + * programs. + */ + tdata = prof_tdata_get(true); + if ((uintptr_t)tdata > (uintptr_t)PROF_TDATA_STATE_MAX) + prof_sample_threshold_update(tctx->tdata); + } + + if ((uintptr_t)tctx > (uintptr_t)1U) { + malloc_mutex_lock(tctx->tdata->lock); + tctx->prepared = false; + if (prof_tctx_should_destroy(tctx)) + prof_tctx_destroy(tctx); + else + malloc_mutex_unlock(tctx->tdata->lock); + } +} + void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) { prof_tctx_set(ptr, tctx); @@ -160,6 +189,7 @@ prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) { tctx->cnts.accumobjs++; tctx->cnts.accumbytes += usize; } + tctx->prepared = false; malloc_mutex_unlock(tctx->tdata->lock); } @@ -529,6 +559,8 @@ prof_tctx_should_destroy(prof_tctx_t *tctx) return (false); if (tctx->cnts.curobjs != 0) return (false); + if (tctx->prepared) + return (false); return (true); } @@ -659,6 +691,8 @@ prof_lookup(prof_bt_t *bt) malloc_mutex_lock(tdata->lock); not_found = ckh_search(&tdata->bt2tctx, bt, NULL, &ret.v); + if (!not_found) /* Note double negative! */ + ret.p->prepared = true; malloc_mutex_unlock(tdata->lock); if (not_found) { void *btkey; @@ -683,6 +717,7 @@ prof_lookup(prof_bt_t *bt) ret.p->tdata = tdata; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); ret.p->gctx = gctx; + ret.p->prepared = true; ret.p->state = prof_tctx_state_nominal; malloc_mutex_lock(tdata->lock); error = ckh_insert(&tdata->bt2tctx, btkey, ret.v); From 61beeb9f69f2f1fd5669b2411245cc7197b5d66a Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 10 Sep 2014 08:49:29 -0700 Subject: [PATCH 101/352] Add sdallocx() to list of functions to prune in pprof. --- bin/pprof | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/pprof b/bin/pprof index 87313f43..5a4c6cd7 100755 --- a/bin/pprof +++ b/bin/pprof @@ -2840,6 +2840,7 @@ sub RemoveUninterestingFrames { 'rallocx', # jemalloc 'xallocx', # jemalloc 'dallocx', # jemalloc + 'sdallocx', # jemalloc 'tc_calloc', 'tc_cfree', 'tc_malloc', From 6b5609d23bf49423fdc6506281e0deac7c3a524e Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Mon, 8 Sep 2014 22:18:49 -0400 Subject: [PATCH 102/352] add likely / unlikely macros --- include/jemalloc/internal/util.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index d2b7a967..82a453d4 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -27,6 +27,14 @@ # define JEMALLOC_CC_SILENCE_INIT(v) #endif +#ifdef __GNUC__ +#define likely(x) __builtin_expect(!!(x), 1) +#define unlikely(x) __builtin_expect(!!(x), 0) +#else +#define likely(x) !!(x) +#define unlikely(x) !!(x) +#endif + /* * Define a custom assert() in order to reduce the chances of deadlock during * assertion failure. From 23fdf8b359a690f457c5300338f4994d06402b95 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Tue, 9 Sep 2014 15:26:05 -0400 Subject: [PATCH 103/352] mark some conditions as unlikely * assertion failure * malloc_init failure * malloc not already initialized (in malloc_init) * running in valgrind * thread cache disabled at runtime Clang and GCC already consider a comparison with NULL or -1 to be cold, so many branches (out-of-memory) are already correctly considered as cold and marking them is not important. --- include/jemalloc/internal/tcache.h | 2 +- include/jemalloc/internal/util.h | 6 ++-- include/jemalloc/internal/valgrind.h | 12 ++++---- src/jemalloc.c | 42 ++++++++++++++-------------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index c0d48b93..292ce461 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -218,7 +218,7 @@ tcache_get(bool create) return (NULL); tcache = *tcache_tsd_get(); - if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) { + if (unlikely((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)) { if (tcache == TCACHE_STATE_DISABLED) return (NULL); tcache = tcache_get_hard(tcache, create); diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index 82a453d4..cc7806d0 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -41,7 +41,7 @@ */ #ifndef assert #define assert(e) do { \ - if (config_debug && !(e)) { \ + if (unlikely(config_debug && !(e))) { \ malloc_printf( \ ": %s:%d: Failed assertion: \"%s\"\n", \ __FILE__, __LINE__, #e); \ @@ -73,14 +73,14 @@ #ifndef assert_not_implemented #define assert_not_implemented(e) do { \ - if (config_debug && !(e)) \ + if (unlikely(config_debug && !(e))) \ not_implemented(); \ } while (0) #endif /* Use to assert a particular configuration, e.g., cassert(config_debug). */ #define cassert(c) do { \ - if ((c) == false) \ + if (unlikely(!(c))) \ not_reached(); \ } while (0) diff --git a/include/jemalloc/internal/valgrind.h b/include/jemalloc/internal/valgrind.h index 52c93f29..a3380df9 100644 --- a/include/jemalloc/internal/valgrind.h +++ b/include/jemalloc/internal/valgrind.h @@ -14,15 +14,15 @@ * usable space. */ #define JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(ptr, usize) do { \ - if (in_valgrind) \ + if (unlikely(in_valgrind)) \ valgrind_make_mem_noaccess(ptr, usize); \ } while (0) #define JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ptr, usize) do { \ - if (in_valgrind) \ + if (unlikely(in_valgrind)) \ valgrind_make_mem_undefined(ptr, usize); \ } while (0) #define JEMALLOC_VALGRIND_MAKE_MEM_DEFINED(ptr, usize) do { \ - if (in_valgrind) \ + if (unlikely(in_valgrind)) \ valgrind_make_mem_defined(ptr, usize); \ } while (0) /* @@ -31,13 +31,13 @@ * Valgrind reports errors, there are no extra stack frames in the backtraces. */ #define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ - if (in_valgrind && cond) \ + if (unlikely(in_valgrind && cond)) \ VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ } while (0) #define JEMALLOC_VALGRIND_REALLOC(maybe_moved, ptr, usize, \ ptr_maybe_null, old_ptr, old_usize, old_rzsize, old_ptr_maybe_null, \ zero) do { \ - if (in_valgrind) { \ + if (unlikely(in_valgrind)) { \ size_t rzsize = p2rz(ptr); \ \ if (!maybe_moved || ptr == old_ptr) { \ @@ -73,7 +73,7 @@ } \ } while (0) #define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ - if (in_valgrind) \ + if (unlikely(in_valgrind)) \ valgrind_freelike_block(ptr, rzsize); \ } while (0) #else diff --git a/src/jemalloc.c b/src/jemalloc.c index 1d4d1a8a..9874361e 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -291,7 +291,7 @@ JEMALLOC_ALWAYS_INLINE_C bool malloc_init(void) { - if (malloc_initialized == false && malloc_init_hard()) + if (unlikely(!malloc_initialized) && malloc_init_hard()) return (true); malloc_thread_init(); @@ -904,7 +904,7 @@ JEMALLOC_ALWAYS_INLINE_C void * imalloc_body(size_t size, size_t *usize) { - if (malloc_init()) + if (unlikely(malloc_init())) return (NULL); if (config_prof && opt_prof) { @@ -912,7 +912,7 @@ imalloc_body(size_t size, size_t *usize) return (imalloc_prof(*usize)); } - if (config_stats || (config_valgrind && in_valgrind)) + if (config_stats || (unlikely(config_valgrind && in_valgrind))) *usize = s2u(size); return (imalloc(size)); } @@ -993,7 +993,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) assert(min_alignment != 0); - if (malloc_init()) { + if (unlikely(malloc_init())) { result = NULL; goto label_oom; } else { @@ -1116,7 +1116,7 @@ je_calloc(size_t num, size_t size) size_t num_size; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - if (malloc_init()) { + if (unlikely(malloc_init())) { num_size = 0; ret = NULL; goto label_return; @@ -1146,7 +1146,7 @@ je_calloc(size_t num, size_t size) usize = s2u(num_size); ret = icalloc_prof(usize); } else { - if (config_stats || (config_valgrind && in_valgrind)) + if (config_stats || unlikely(config_valgrind && in_valgrind)) usize = s2u(num_size); ret = icalloc(num_size); } @@ -1222,7 +1222,7 @@ ifree(void *ptr, bool try_tcache) usize = isalloc(ptr, config_prof); if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - if (config_valgrind && in_valgrind) + if (unlikely(config_valgrind && in_valgrind)) rzsize = p2rz(ptr); iqalloc(ptr, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); @@ -1240,7 +1240,7 @@ isfree(void *ptr, size_t usize, bool try_tcache) prof_free(ptr, usize); if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - if (config_valgrind && in_valgrind) + if (unlikely(config_valgrind && in_valgrind)) rzsize = p2rz(ptr); isqalloc(ptr, usize, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); @@ -1269,16 +1269,16 @@ je_realloc(void *ptr, size_t size) malloc_thread_init(); if ((config_prof && opt_prof) || config_stats || - (config_valgrind && in_valgrind)) + unlikely(config_valgrind && in_valgrind)) old_usize = isalloc(ptr, config_prof); - if (config_valgrind && in_valgrind) + if (unlikely(config_valgrind && in_valgrind)) old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); if (config_prof && opt_prof) { usize = s2u(size); ret = irealloc_prof(ptr, old_usize, usize); } else { - if (config_stats || (config_valgrind && in_valgrind)) + if (config_stats || unlikely(config_valgrind && in_valgrind)) usize = s2u(size); ret = iralloc(ptr, size, 0, false); } @@ -1506,7 +1506,7 @@ imallocx_no_prof(size_t size, int flags, size_t *usize) arena_t *arena; if (flags == 0) { - if (config_stats || (config_valgrind && in_valgrind)) + if (config_stats || unlikely(config_valgrind && in_valgrind)) *usize = s2u(size); return (imalloc(size)); } @@ -1524,7 +1524,7 @@ je_mallocx(size_t size, int flags) assert(size != 0); - if (malloc_init()) + if (unlikely(malloc_init())) goto label_oom; if (config_prof && opt_prof) @@ -1642,9 +1642,9 @@ je_rallocx(void *ptr, size_t size, int flags) } if ((config_prof && opt_prof) || config_stats || - (config_valgrind && in_valgrind)) + (unlikely(config_valgrind && in_valgrind))) old_usize = isalloc(ptr, config_prof); - if (config_valgrind && in_valgrind) + if (unlikely(config_valgrind && in_valgrind)) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { @@ -1777,7 +1777,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) arena = NULL; old_usize = isalloc(ptr, config_prof); - if (config_valgrind && in_valgrind) + if (unlikely(config_valgrind && in_valgrind)) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { @@ -1883,7 +1883,7 @@ je_nallocx(size_t size, int flags) assert(size != 0); - if (malloc_init()) + if (unlikely(malloc_init())) return (0); return (inallocx(size, flags)); @@ -1894,7 +1894,7 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - if (malloc_init()) + if (unlikely(malloc_init())) return (EAGAIN); return (ctl_byname(name, oldp, oldlenp, newp, newlen)); @@ -1904,7 +1904,7 @@ int je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { - if (malloc_init()) + if (unlikely(malloc_init())) return (EAGAIN); return (ctl_nametomib(name, mibp, miblenp)); @@ -1915,7 +1915,7 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { - if (malloc_init()) + if (unlikely(malloc_init())) return (EAGAIN); return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); @@ -2064,7 +2064,7 @@ static void * a0alloc(size_t size, bool zero) { - if (malloc_init()) + if (unlikely(malloc_init())) return (NULL); if (size == 0) From 91566fc079cfaeaf2b424b7f40d6b9d8669d0470 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 11 Sep 2014 13:15:33 -0700 Subject: [PATCH 104/352] Fix mallocx() to always honor MALLOCX_ARENA() when profiling. --- src/jemalloc.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 9874361e..f6be7514 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1435,7 +1435,6 @@ imallocx_flags(size_t usize, size_t alignment, bool zero, bool try_tcache, return (imalloct(usize, try_tcache, arena)); } - JEMALLOC_ALWAYS_INLINE_C void * imallocx_maybe_flags(size_t size, int flags, size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena) @@ -1455,7 +1454,7 @@ imallocx_prof_sample(size_t size, int flags, size_t usize, size_t alignment, if (usize <= SMALL_MAXCLASS) { assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); - p = imalloc(LARGE_MINCLASS); + p = imalloct(LARGE_MINCLASS, try_tcache, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); From 9c640bfdd4e2f25180a32ed3704ce8e4c4cc21f1 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 11 Sep 2014 16:20:44 -0700 Subject: [PATCH 105/352] Apply likely()/unlikely() to allocation/deallocation fast paths. --- include/jemalloc/internal/arena.h | 52 +++---- .../jemalloc/internal/jemalloc_internal.h.in | 4 +- include/jemalloc/internal/prof.h | 11 +- include/jemalloc/internal/tcache.h | 32 ++--- src/arena.c | 28 ++-- src/huge.c | 6 +- src/jemalloc.c | 130 +++++++++--------- src/quarantine.c | 4 +- 8 files changed, 138 insertions(+), 129 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 6ab0ae71..bfb0b3cf 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -488,7 +488,8 @@ void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); -void arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache); +void arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, + bool try_tcache); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) @@ -539,7 +540,7 @@ small_size2bin(size_t size) { assert(size > 0); - if (size <= LOOKUP_MAXCLASS) + if (likely(size <= LOOKUP_MAXCLASS)) return (small_size2bin_lookup(size)); else return (small_size2bin_compute(size)); @@ -627,7 +628,7 @@ small_s2u(size_t size) { assert(size > 0); - if (size <= LOOKUP_MAXCLASS) + if (likely(size <= LOOKUP_MAXCLASS)) return (small_s2u_lookup(size)); else return (small_s2u_compute(size)); @@ -864,7 +865,7 @@ arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) cassert(config_prof); - if (prof_interval == 0) + if (likely(prof_interval == 0)) return (false); return (arena_prof_accum_impl(arena, accumbytes)); } @@ -875,7 +876,7 @@ arena_prof_accum(arena_t *arena, uint64_t accumbytes) cassert(config_prof); - if (prof_interval == 0) + if (likely(prof_interval == 0)) return (false); { @@ -995,8 +996,8 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) }; - if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + - 2)) { + if (likely(interval <= ((sizeof(interval_invs) / + sizeof(unsigned)) + 2))) { regind = (diff * interval_invs[interval - 3]) >> SIZE_INV_SHIFT; } else @@ -1025,7 +1026,7 @@ arena_prof_tctx_get(const void *ptr) pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) + if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) ret = (prof_tctx_t *)(uintptr_t)1U; else ret = arena_miscelm_get(chunk, pageind)->prof_tctx; @@ -1047,7 +1048,7 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (arena_mapbits_large_get(chunk, pageind) != 0) + if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) arena_miscelm_get(chunk, pageind)->prof_tctx = tctx; } @@ -1059,8 +1060,9 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) assert(size != 0); assert(size <= arena_maxclass); - if (size <= SMALL_MAXCLASS) { - if (try_tcache && (tcache = tcache_get(true)) != NULL) + if (likely(size <= SMALL_MAXCLASS)) { + if (likely(try_tcache) && likely((tcache = tcache_get(true)) != + NULL)) return (tcache_alloc_small(tcache, size, zero)); else { return (arena_malloc_small(choose_arena(arena), size, @@ -1071,8 +1073,8 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) * Initialize tcache after checking size in order to avoid * infinite recursion during tcache initialization. */ - if (try_tcache && size <= tcache_maxclass && (tcache = - tcache_get(true)) != NULL) + if (try_tcache && size <= tcache_maxclass && likely((tcache = + tcache_get(true)) != NULL)) return (tcache_alloc_large(tcache, size, zero)); else { return (arena_malloc_large(choose_arena(arena), size, @@ -1096,8 +1098,8 @@ arena_salloc(const void *ptr, bool demote) pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; assert(arena_mapbits_allocated_get(chunk, pageind) != 0); binind = arena_mapbits_binind_get(chunk, pageind); - if (binind == BININD_INVALID || (config_prof && demote == false && - arena_mapbits_large_get(chunk, pageind) != 0)) { + if (unlikely(binind == BININD_INVALID || (config_prof && demote == false + && arena_mapbits_large_get(chunk, pageind) != 0))) { /* * Large allocation. In the common case (demote == true), and * as this is an inline function, most callers will only end up @@ -1137,10 +1139,12 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; mapbits = arena_mapbits_get(chunk, pageind); assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { + if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { /* Small allocation. */ - if (try_tcache && (tcache = tcache_get(false)) != NULL) { - size_t binind = arena_ptr_small_binind_get(ptr, mapbits); + if (likely(try_tcache) && likely((tcache = tcache_get(false)) != + NULL)) { + size_t binind = arena_ptr_small_binind_get(ptr, + mapbits); tcache_dalloc_small(tcache, ptr, binind); } else arena_dalloc_small(chunk->arena, chunk, ptr, pageind); @@ -1149,8 +1153,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) assert(((uintptr_t)ptr & PAGE_MASK) == 0); - if (try_tcache && size <= tcache_maxclass && (tcache = - tcache_get(false)) != NULL) { + if (try_tcache && size <= tcache_maxclass && likely((tcache = + tcache_get(false)) != NULL)) { tcache_dalloc_large(tcache, ptr, size); } else arena_dalloc_large(chunk->arena, chunk, ptr); @@ -1165,13 +1169,15 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache) assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - if (size < PAGE) { + if (likely(size <= SMALL_MAXCLASS)) { /* Small allocation. */ - if (try_tcache && (tcache = tcache_get(false)) != NULL) { + if (likely(try_tcache) && likely((tcache = tcache_get(false)) != + NULL)) { size_t binind = small_size2bin(size); tcache_dalloc_small(tcache, ptr, binind); } else { - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> + LG_PAGE; arena_dalloc_small(chunk->arena, chunk, ptr, pageind); } } else { diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 81d46fc3..a380a414 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -814,7 +814,7 @@ JEMALLOC_ALWAYS_INLINE void iqalloc(void *ptr, bool try_tcache) { - if (config_fill && opt_quarantine) + if (config_fill && unlikely(opt_quarantine)) quarantine(ptr); else idalloct(ptr, try_tcache); @@ -824,7 +824,7 @@ JEMALLOC_ALWAYS_INLINE void isqalloc(void *ptr, size_t size, bool try_tcache) { - if (config_fill && opt_quarantine) + if (config_fill && unlikely(opt_quarantine)) quarantine(ptr); else isdalloct(ptr, size, try_tcache); diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 920ec63f..a1e7ac5e 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -400,7 +400,8 @@ prof_alloc_prep(size_t usize, bool update) assert(usize == s2u(usize)); - if (!opt_prof_active || prof_sample_accum_update(usize, update, &tdata)) + if (!opt_prof_active || likely(prof_sample_accum_update(usize, update, + &tdata))) ret = (prof_tctx_t *)(uintptr_t)1U; else { bt_init(&bt, tdata->vec); @@ -419,7 +420,7 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) assert(ptr != NULL); assert(usize == isalloc(ptr, true)); - if ((uintptr_t)tctx > (uintptr_t)1U) + if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) prof_malloc_sample_object(ptr, usize, tctx); else prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); @@ -447,9 +448,9 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated, } } - if ((uintptr_t)old_tctx > (uintptr_t)1U) + if (unlikely((uintptr_t)old_tctx > (uintptr_t)1U)) prof_free_sampled_object(old_usize, old_tctx); - if ((uintptr_t)tctx > (uintptr_t)1U) + if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) prof_malloc_sample_object(ptr, usize, tctx); else prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); @@ -463,7 +464,7 @@ prof_free(const void *ptr, size_t usize) cassert(config_prof); assert(usize == isalloc(ptr, true)); - if ((uintptr_t)tctx > (uintptr_t)1U) + if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) prof_free_sampled_object(usize, tctx); } #endif diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 292ce461..c9d723aa 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -236,7 +236,7 @@ tcache_event(tcache_t *tcache) tcache->ev_cnt++; assert(tcache->ev_cnt <= TCACHE_GC_INCR); - if (tcache->ev_cnt == TCACHE_GC_INCR) + if (unlikely(tcache->ev_cnt == TCACHE_GC_INCR)) tcache_event_hard(tcache); } @@ -245,12 +245,12 @@ tcache_alloc_easy(tcache_bin_t *tbin) { void *ret; - if (tbin->ncached == 0) { + if (unlikely(tbin->ncached == 0)) { tbin->low_water = -1; return (NULL); } tbin->ncached--; - if ((int)tbin->ncached < tbin->low_water) + if (unlikely((int)tbin->ncached < tbin->low_water)) tbin->low_water = tbin->ncached; ret = tbin->avail[tbin->ncached]; return (ret); @@ -268,23 +268,23 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) tbin = &tcache->tbins[binind]; size = small_bin2size(binind); ret = tcache_alloc_easy(tbin); - if (ret == NULL) { + if (unlikely(ret == NULL)) { ret = tcache_alloc_small_hard(tcache, tbin, binind); if (ret == NULL) return (NULL); } assert(tcache_salloc(ret) == size); - if (zero == false) { + if (likely(zero == false)) { if (config_fill) { - if (opt_junk) { + if (unlikely(opt_junk)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], false); - } else if (opt_zero) + } else if (unlikely(opt_zero)) memset(ret, 0, size); } } else { - if (config_fill && opt_junk) { + if (config_fill && unlikely(opt_junk)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } @@ -312,7 +312,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) assert(binind < nhbins); tbin = &tcache->tbins[binind]; ret = tcache_alloc_easy(tbin); - if (ret == NULL) { + if (unlikely(ret == NULL)) { /* * Only allocate one large object at a time, because it's quite * expensive to create one and not use it. @@ -329,11 +329,11 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) arena_mapbits_large_binind_set(chunk, pageind, BININD_INVALID); } - if (zero == false) { + if (likely(zero == false)) { if (config_fill) { - if (opt_junk) + if (unlikely(opt_junk)) memset(ret, 0xa5, size); - else if (opt_zero) + else if (unlikely(opt_zero)) memset(ret, 0, size); } } else @@ -357,12 +357,12 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind) assert(tcache_salloc(ptr) <= SMALL_MAXCLASS); - if (config_fill && opt_junk) + if (config_fill && unlikely(opt_junk)) arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; - if (tbin->ncached == tbin_info->ncached_max) { + if (unlikely(tbin->ncached == tbin_info->ncached_max)) { tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >> 1), tcache); } @@ -386,12 +386,12 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) binind = NBINS + (size >> LG_PAGE) - 1; - if (config_fill && opt_junk) + if (config_fill && unlikely(opt_junk)) memset(ptr, 0x5a, size); tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; - if (tbin->ncached == tbin_info->ncached_max) { + if (unlikely(tbin->ncached == tbin_info->ncached_max)) { tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >> 1), tcache); } diff --git a/src/arena.c b/src/arena.c index 8d34cf60..35d792a2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1365,7 +1365,7 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, ptr = arena_bin_malloc_hard(arena, bin); if (ptr == NULL) break; - if (config_fill && opt_junk) { + if (config_fill && unlikely(opt_junk)) { arena_alloc_junk_small(ptr, &arena_bin_info[binind], true); } @@ -1519,15 +1519,15 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) if (zero == false) { if (config_fill) { - if (opt_junk) { + if (unlikely(opt_junk)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], false); - } else if (opt_zero) + } else if (unlikely(opt_zero)) memset(ret, 0, size); } JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { - if (config_fill && opt_junk) { + if (config_fill && unlikely(opt_junk)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } @@ -1568,9 +1568,9 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) if (zero == false) { if (config_fill) { - if (opt_junk) + if (unlikely(opt_junk)) memset(ret, 0xa5, size); - else if (opt_zero) + else if (unlikely(opt_zero)) memset(ret, 0, size); } } @@ -1626,9 +1626,9 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) malloc_mutex_unlock(&arena->lock); if (config_fill && zero == false) { - if (opt_junk) + if (unlikely(opt_junk)) memset(ret, 0xa5, size); - else if (opt_zero) + else if (unlikely(opt_zero)) memset(ret, 0, size); } return (ret); @@ -1771,7 +1771,7 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, if (config_fill || config_stats) size = bin_info->reg_size; - if (config_fill && opt_junk) + if (config_fill && unlikely(opt_junk)) arena_dalloc_junk_small(ptr, bin_info); arena_run_reg_dalloc(run, ptr); @@ -1825,7 +1825,7 @@ static void arena_dalloc_junk_large(void *ptr, size_t usize) { - if (config_fill && opt_junk) + if (config_fill && unlikely(opt_junk)) memset(ptr, 0x5a, usize); } #ifdef JEMALLOC_JET @@ -1967,7 +1967,7 @@ static void arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) { - if (config_fill && opt_junk) { + if (config_fill && unlikely(opt_junk)) { memset((void *)((uintptr_t)ptr + usize), 0x5a, old_usize - usize); } @@ -2011,11 +2011,11 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, oldsize, PAGE_CEILING(size), psize - PAGE_CEILING(size), zero); if (config_fill && ret == false && zero == false) { - if (opt_junk) { + if (unlikely(opt_junk)) { memset((void *)((uintptr_t)ptr + oldsize), 0xa5, isalloc(ptr, config_prof) - oldsize); - } else if (opt_zero) { + } else if (unlikely(opt_zero)) { memset((void *)((uintptr_t)ptr + oldsize), 0, isalloc(ptr, config_prof) - oldsize); @@ -2272,7 +2272,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) * minimum alignment; without the padding, each redzone would have to * be twice as large in order to maintain alignment. */ - if (config_fill && opt_redzone) { + if (config_fill && unlikely(opt_redzone)) { size_t align_min = ZU(1) << (jemalloc_ffs(bin_info->reg_size) - 1); if (align_min <= REDZONE_MINSIZE) { diff --git a/src/huge.c b/src/huge.c index e7733093..0b7db7fc 100644 --- a/src/huge.c +++ b/src/huge.c @@ -62,9 +62,9 @@ huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) malloc_mutex_unlock(&huge_mtx); if (config_fill && zero == false) { - if (opt_junk) + if (unlikely(opt_junk)) memset(ret, 0xa5, csize); - else if (opt_zero && is_zeroed == false) + else if (unlikely(opt_zero) && is_zeroed == false) memset(ret, 0, csize); } @@ -141,7 +141,7 @@ static void huge_dalloc_junk(void *ptr, size_t usize) { - if (config_fill && have_dss && opt_junk) { + if (config_fill && have_dss && unlikely(opt_junk)) { /* * Only bother junk filling if the chunk isn't about to be * unmapped. diff --git a/src/jemalloc.c b/src/jemalloc.c index f6be7514..dfb12666 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -87,7 +87,7 @@ typedef struct { #ifdef JEMALLOC_UTRACE # define UTRACE(a, b, c) do { \ - if (opt_utrace) { \ + if (unlikely(opt_utrace)) { \ int utrace_serrno = errno; \ malloc_utrace_t ut; \ ut.p = (a); \ @@ -283,7 +283,7 @@ malloc_thread_init(void) * a best effort attempt at initializing its TSD by hooking all * allocation events. */ - if (config_fill && opt_quarantine) + if (config_fill && unlikely(opt_quarantine)) quarantine_alloc_hook(); } @@ -397,13 +397,13 @@ malloc_conf_init(void) */ if (config_valgrind) { in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; - if (config_fill && in_valgrind) { + if (config_fill && unlikely(in_valgrind)) { opt_junk = false; assert(opt_zero == false); opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; opt_redzone = true; } - if (config_tcache && in_valgrind) + if (config_tcache && unlikely(in_valgrind)) opt_tcache = false; } @@ -887,7 +887,7 @@ imalloc_prof(size_t usize) prof_tctx_t *tctx; tctx = prof_alloc_prep(usize, true); - if ((uintptr_t)tctx != (uintptr_t)1U) + if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = imalloc_prof_sample(usize, tctx); else p = imalloc(usize); @@ -912,7 +912,7 @@ imalloc_body(size_t size, size_t *usize) return (imalloc_prof(*usize)); } - if (config_stats || (unlikely(config_valgrind && in_valgrind))) + if (config_stats || (config_valgrind && unlikely(in_valgrind))) *usize = s2u(size); return (imalloc(size)); } @@ -927,15 +927,15 @@ je_malloc(size_t size) size = 1; ret = imalloc_body(size, &usize); - if (ret == NULL) { - if (config_xmalloc && opt_xmalloc) { + if (unlikely(ret == NULL)) { + if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error in malloc(): " "out of memory\n"); abort(); } set_errno(ENOMEM); } - if (config_stats && ret != NULL) { + if (config_stats && likely(ret != NULL)) { assert(usize == isalloc(ret, config_prof)); thread_allocated_tsd_get()->allocated += usize; } @@ -970,7 +970,7 @@ imemalign_prof(size_t alignment, size_t usize) prof_tctx_t *tctx; tctx = prof_alloc_prep(usize, true); - if ((uintptr_t)tctx != (uintptr_t)1U) + if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = imemalign_prof_sample(alignment, usize, tctx); else p = ipalloc(usize, alignment, false); @@ -1001,9 +1001,9 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) size = 1; /* Make sure that alignment is a large enough power of 2. */ - if (((alignment - 1) & alignment) != 0 - || (alignment < min_alignment)) { - if (config_xmalloc && opt_xmalloc) { + if (unlikely(((alignment - 1) & alignment) != 0 + || (alignment < min_alignment))) { + if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error allocating " "aligned memory: invalid alignment\n"); abort(); @@ -1014,7 +1014,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) } usize = sa2u(size, alignment); - if (usize == 0) { + if (unlikely(usize == 0)) { result = NULL; goto label_oom; } @@ -1023,14 +1023,14 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) result = imemalign_prof(alignment, usize); else result = ipalloc(usize, alignment, false); - if (result == NULL) + if (unlikely(result == NULL)) goto label_oom; } *memptr = result; ret = 0; label_return: - if (config_stats && result != NULL) { + if (config_stats && likely(result != NULL)) { assert(usize == isalloc(result, config_prof)); thread_allocated_tsd_get()->allocated += usize; } @@ -1038,7 +1038,7 @@ label_return: return (ret); label_oom: assert(result == NULL); - if (config_xmalloc && opt_xmalloc) { + if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error allocating aligned memory: " "out of memory\n"); abort(); @@ -1062,7 +1062,7 @@ je_aligned_alloc(size_t alignment, size_t size) void *ret; int err; - if ((err = imemalign(&ret, alignment, size, 1)) != 0) { + if (unlikely((err = imemalign(&ret, alignment, size, 1)) != 0)) { ret = NULL; set_errno(err); } @@ -1096,7 +1096,7 @@ icalloc_prof(size_t usize) prof_tctx_t *tctx; tctx = prof_alloc_prep(usize, true); - if ((uintptr_t)tctx != (uintptr_t)1U) + if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = icalloc_prof_sample(usize, tctx); else p = icalloc(usize); @@ -1123,7 +1123,7 @@ je_calloc(size_t num, size_t size) } num_size = num * size; - if (num_size == 0) { + if (unlikely(num_size == 0)) { if (num == 0 || size == 0) num_size = 1; else { @@ -1135,8 +1135,8 @@ je_calloc(size_t num, size_t size) * overflow during multiplication if neither operand uses any of the * most significant half of the bits in a size_t. */ - } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) - && (num_size / size != num)) { + } else if (unlikely(((num | size) & (SIZE_T_MAX << (sizeof(size_t) << + 2))) && (num_size / size != num))) { /* size_t overflow. */ ret = NULL; goto label_return; @@ -1146,21 +1146,21 @@ je_calloc(size_t num, size_t size) usize = s2u(num_size); ret = icalloc_prof(usize); } else { - if (config_stats || unlikely(config_valgrind && in_valgrind)) + if (config_stats || (config_valgrind && unlikely(in_valgrind))) usize = s2u(num_size); ret = icalloc(num_size); } label_return: - if (ret == NULL) { - if (config_xmalloc && opt_xmalloc) { + if (unlikely(ret == NULL)) { + if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error in calloc(): out of " "memory\n"); abort(); } set_errno(ENOMEM); } - if (config_stats && ret != NULL) { + if (config_stats && likely(ret != NULL)) { assert(usize == isalloc(ret, config_prof)); thread_allocated_tsd_get()->allocated += usize; } @@ -1195,7 +1195,7 @@ irealloc_prof(void *oldptr, size_t old_usize, size_t usize) old_tctx = prof_tctx_get(oldptr); tctx = prof_alloc_prep(usize, true); - if ((uintptr_t)tctx != (uintptr_t)1U) + if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) p = irealloc_prof_sample(oldptr, usize, tctx); else p = iralloc(oldptr, usize, 0, false); @@ -1222,7 +1222,7 @@ ifree(void *ptr, bool try_tcache) usize = isalloc(ptr, config_prof); if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - if (unlikely(config_valgrind && in_valgrind)) + if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); iqalloc(ptr, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); @@ -1240,7 +1240,7 @@ isfree(void *ptr, size_t usize, bool try_tcache) prof_free(ptr, usize); if (config_stats) thread_allocated_tsd_get()->deallocated += usize; - if (unlikely(config_valgrind && in_valgrind)) + if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); isqalloc(ptr, usize, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); @@ -1254,7 +1254,7 @@ je_realloc(void *ptr, size_t size) size_t old_usize = 0; UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); - if (size == 0) { + if (unlikely(size == 0)) { if (ptr != NULL) { /* realloc(ptr, 0) is equivalent to free(ptr). */ UTRACE(ptr, 0, 0); @@ -1264,21 +1264,22 @@ je_realloc(void *ptr, size_t size) size = 1; } - if (ptr != NULL) { + if (likely(ptr != NULL)) { assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); if ((config_prof && opt_prof) || config_stats || - unlikely(config_valgrind && in_valgrind)) + (config_valgrind && unlikely(in_valgrind))) old_usize = isalloc(ptr, config_prof); - if (unlikely(config_valgrind && in_valgrind)) + if (config_valgrind && unlikely(in_valgrind)) old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); if (config_prof && opt_prof) { usize = s2u(size); ret = irealloc_prof(ptr, old_usize, usize); } else { - if (config_stats || unlikely(config_valgrind && in_valgrind)) + if (config_stats || (config_valgrind && + unlikely(in_valgrind))) usize = s2u(size); ret = iralloc(ptr, size, 0, false); } @@ -1287,15 +1288,15 @@ je_realloc(void *ptr, size_t size) ret = imalloc_body(size, &usize); } - if (ret == NULL) { - if (config_xmalloc && opt_xmalloc) { + if (unlikely(ret == NULL)) { + if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error in realloc(): " "out of memory\n"); abort(); } set_errno(ENOMEM); } - if (config_stats && ret != NULL) { + if (config_stats && likely(ret != NULL)) { thread_allocated_t *ta; assert(usize == isalloc(ret, config_prof)); ta = thread_allocated_tsd_get(); @@ -1313,7 +1314,7 @@ je_free(void *ptr) { UTRACE(ptr, 0, 0); - if (ptr != NULL) + if (likely(ptr != NULL)) ifree(ptr, true); } @@ -1410,7 +1411,7 @@ imallocx_flags_decode(size_t size, int flags, size_t *usize, size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) { - if (flags == 0) { + if (likely(flags == 0)) { *usize = s2u(size); assert(usize != 0); *alignment = 0; @@ -1440,7 +1441,7 @@ imallocx_maybe_flags(size_t size, int flags, size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena) { - if (flags == 0) + if (likely(flags == 0)) return (imalloc(size)); return (imallocx_flags(usize, alignment, zero, try_tcache, arena)); } @@ -1479,7 +1480,7 @@ imallocx_prof(size_t size, int flags, size_t *usize) imallocx_flags_decode(size, flags, usize, &alignment, &zero, &try_tcache, &arena); tctx = prof_alloc_prep(*usize, true); - if ((uintptr_t)tctx == (uintptr_t)1U) { + if (likely((uintptr_t)tctx == (uintptr_t)1U)) { p = imallocx_maybe_flags(size, flags, *usize, alignment, zero, try_tcache, arena); } else if ((uintptr_t)tctx > (uintptr_t)1U) { @@ -1487,7 +1488,7 @@ imallocx_prof(size_t size, int flags, size_t *usize) try_tcache, arena); } else p = NULL; - if (p == NULL) { + if (unlikely(p == NULL)) { prof_alloc_rollback(tctx, true); return (NULL); } @@ -1504,8 +1505,8 @@ imallocx_no_prof(size_t size, int flags, size_t *usize) bool try_tcache; arena_t *arena; - if (flags == 0) { - if (config_stats || unlikely(config_valgrind && in_valgrind)) + if (likely(flags == 0)) { + if (config_stats || (config_valgrind && unlikely(in_valgrind))) *usize = s2u(size); return (imalloc(size)); } @@ -1530,7 +1531,7 @@ je_mallocx(size_t size, int flags) p = imallocx_prof(size, flags, &usize); else p = imallocx_no_prof(size, flags, &usize); - if (p == NULL) + if (unlikely(p == NULL)) goto label_oom; if (config_stats) { @@ -1541,7 +1542,7 @@ je_mallocx(size_t size, int flags) JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags)); return (p); label_oom: - if (config_xmalloc && opt_xmalloc) { + if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error in mallocx(): out of memory\n"); abort(); } @@ -1582,14 +1583,14 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, old_tctx = prof_tctx_get(oldptr); tctx = prof_alloc_prep(*usize, true); - if ((uintptr_t)tctx != (uintptr_t)1U) { + if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx); } else { p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); } - if (p == NULL) { + if (unlikely(p == NULL)) { prof_alloc_rollback(tctx, true); return (NULL); } @@ -1614,7 +1615,8 @@ void * je_rallocx(void *ptr, size_t size, int flags) { void *p; - size_t usize, old_usize; + size_t usize; + UNUSED size_t old_usize JEMALLOC_CC_SILENCE_INIT(0); UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t alignment = MALLOCX_ALIGN_GET(flags); bool zero = flags & MALLOCX_ZERO; @@ -1626,7 +1628,7 @@ je_rallocx(void *ptr, size_t size, int flags) assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); - if ((flags & MALLOCX_ARENA_MASK) != 0) { + if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk; try_tcache_alloc = false; @@ -1641,9 +1643,9 @@ je_rallocx(void *ptr, size_t size, int flags) } if ((config_prof && opt_prof) || config_stats || - (unlikely(config_valgrind && in_valgrind))) + ((config_valgrind && unlikely(in_valgrind)))) old_usize = isalloc(ptr, config_prof); - if (unlikely(config_valgrind && in_valgrind)) + if (config_valgrind && unlikely(in_valgrind)) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { @@ -1651,14 +1653,14 @@ je_rallocx(void *ptr, size_t size, int flags) assert(usize != 0); p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, try_tcache_alloc, try_tcache_dalloc, arena); - if (p == NULL) + if (unlikely(p == NULL)) goto label_oom; } else { p = iralloct(ptr, size, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); - if (p == NULL) + if (unlikely(p == NULL)) goto label_oom; - if (config_stats || (config_valgrind && in_valgrind)) + if (config_stats || (config_valgrind && unlikely(in_valgrind))) usize = isalloc(p, config_prof); } @@ -1673,7 +1675,7 @@ je_rallocx(void *ptr, size_t size, int flags) old_rzsize, false, zero); return (p); label_oom: - if (config_xmalloc && opt_xmalloc) { + if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error in rallocx(): out of memory\n"); abort(); } @@ -1738,14 +1740,14 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment); tctx = prof_alloc_prep(max_usize, false); - if ((uintptr_t)tctx != (uintptr_t)1U) { + if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, alignment, zero, max_usize, arena, tctx); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero, arena); } - if (usize == old_usize) { + if (unlikely(usize == old_usize)) { prof_alloc_rollback(tctx, false); return (usize); } @@ -1769,14 +1771,14 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); - if ((flags & MALLOCX_ARENA_MASK) != 0) { + if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena = arenas[arena_ind]; } else arena = NULL; old_usize = isalloc(ptr, config_prof); - if (unlikely(config_valgrind && in_valgrind)) + if (config_valgrind && unlikely(in_valgrind)) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { @@ -1786,7 +1788,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero, arena); } - if (usize == old_usize) + if (unlikely(usize == old_usize)) goto label_not_resized; if (config_stats) { @@ -1828,7 +1830,7 @@ je_dallocx(void *ptr, int flags) assert(ptr != NULL); assert(malloc_initialized || IS_INITIALIZER); - if ((flags & MALLOCX_ARENA_MASK) != 0) { + if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); try_tcache = (chunk == ptr || chunk->arena != @@ -1845,7 +1847,7 @@ inallocx(size_t size, int flags) { size_t usize; - if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) + if (likely((flags & MALLOCX_LG_ALIGN_MASK) == 0)) usize = s2u(size); else usize = sa2u(size, MALLOCX_ALIGN_GET_SPECIFIED(flags)); @@ -1864,7 +1866,7 @@ je_sdallocx(void *ptr, size_t size, int flags) usize = inallocx(size, flags); assert(usize == isalloc(ptr, config_prof)); - if ((flags & MALLOCX_ARENA_MASK) != 0) { + if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); try_tcache = (chunk == ptr || chunk->arena != diff --git a/src/quarantine.c b/src/quarantine.c index 3b874422..efddeae7 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -141,12 +141,12 @@ quarantine(void *ptr) obj->usize = usize; quarantine->curbytes += usize; quarantine->curobjs++; - if (config_fill && opt_junk) { + if (config_fill && unlikely(opt_junk)) { /* * Only do redzone validation if Valgrind isn't in * operation. */ - if ((config_valgrind == false || in_valgrind == false) + if ((!config_valgrind || likely(!in_valgrind)) && usize <= SMALL_MAXCLASS) arena_quarantine_junk_small(ptr, usize); else From c3e9e7b0412e97e4976507f914fd39901b023537 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 11 Sep 2014 17:04:03 -0700 Subject: [PATCH 106/352] Fix irallocx_prof() sample logic. Fix irallocx_prof() sample logic to only update the threshold counter after it knows what size the allocation ended up being. This regression was caused by 6e73dc194ee9682d3eacaf725a989f04629718f7 (Fix a profile sampling race.), which did not make it into any releases prior to this fix. --- src/jemalloc.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index dfb12666..c5b8f520 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1582,7 +1582,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(oldptr); - tctx = prof_alloc_prep(*usize, true); + tctx = prof_alloc_prep(*usize, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx); @@ -1591,7 +1591,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, try_tcache_dalloc, arena); } if (unlikely(p == NULL)) { - prof_alloc_rollback(tctx, true); + prof_alloc_rollback(tctx, false); return (NULL); } @@ -1606,7 +1606,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, */ *usize = isalloc(p, config_prof); } - prof_realloc(p, *usize, tctx, true, old_usize, old_tctx); + prof_realloc(p, *usize, tctx, false, old_usize, old_tctx); return (p); } From 9d8f3d203327a7ee9ba92814e1fd8a7d1b9c421b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 11 Sep 2014 18:06:30 -0700 Subject: [PATCH 107/352] Fix prof regressions. Don't use atomic_add_uint64(), because it isn't available on 32-bit platforms. Fix forking support functions to manage all prof-related mutexes. These regressions were introduced by 602c8e0971160e4b85b08b16cf8a2375aa24bc04 (Implement per thread heap profiling.), which did not make it into any releases prior to these fixes. --- src/prof.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/prof.c b/src/prof.c index 9495afc4..a773e224 100644 --- a/src/prof.c +++ b/src/prof.c @@ -68,6 +68,7 @@ static prof_tdata_tree_t tdatas; static malloc_mutex_t tdatas_mtx; static uint64_t next_thr_uid; +static malloc_mutex_t next_thr_uid_mtx; static malloc_mutex_t prof_dump_seq_mtx; static uint64_t prof_dump_seq; @@ -1498,8 +1499,14 @@ prof_bt_keycomp(const void *k1, const void *k2) JEMALLOC_INLINE_C uint64_t prof_thr_uid_alloc(void) { + uint64_t thr_uid; - return (atomic_add_uint64(&next_thr_uid, 1) - 1); + malloc_mutex_lock(&next_thr_uid_mtx); + thr_uid = next_thr_uid; + next_thr_uid++; + malloc_mutex_unlock(&next_thr_uid_mtx); + + return (thr_uid); } static prof_tdata_t * @@ -1785,6 +1792,8 @@ prof_boot2(void) return (true); next_thr_uid = 0; + if (malloc_mutex_init(&next_thr_uid_mtx)) + return (true); if (malloc_mutex_init(&prof_dump_seq_mtx)) return (true); @@ -1836,10 +1845,14 @@ prof_prefork(void) if (opt_prof) { unsigned i; + malloc_mutex_prefork(&tdatas_mtx); malloc_mutex_prefork(&bt2gctx_mtx); + malloc_mutex_prefork(&next_thr_uid_mtx); malloc_mutex_prefork(&prof_dump_seq_mtx); for (i = 0; i < PROF_NCTX_LOCKS; i++) malloc_mutex_prefork(&gctx_locks[i]); + for (i = 0; i < PROF_NTDATA_LOCKS; i++) + malloc_mutex_prefork(&tdata_locks[i]); } } @@ -1850,10 +1863,14 @@ prof_postfork_parent(void) if (opt_prof) { unsigned i; + for (i = 0; i < PROF_NTDATA_LOCKS; i++) + malloc_mutex_postfork_parent(&tdata_locks[i]); for (i = 0; i < PROF_NCTX_LOCKS; i++) malloc_mutex_postfork_parent(&gctx_locks[i]); malloc_mutex_postfork_parent(&prof_dump_seq_mtx); + malloc_mutex_postfork_parent(&next_thr_uid_mtx); malloc_mutex_postfork_parent(&bt2gctx_mtx); + malloc_mutex_postfork_parent(&tdatas_mtx); } } @@ -1864,10 +1881,14 @@ prof_postfork_child(void) if (opt_prof) { unsigned i; + for (i = 0; i < PROF_NTDATA_LOCKS; i++) + malloc_mutex_postfork_child(&tdata_locks[i]); for (i = 0; i < PROF_NCTX_LOCKS; i++) malloc_mutex_postfork_child(&gctx_locks[i]); malloc_mutex_postfork_child(&prof_dump_seq_mtx); + malloc_mutex_postfork_child(&next_thr_uid_mtx); malloc_mutex_postfork_child(&bt2gctx_mtx); + malloc_mutex_postfork_child(&tdatas_mtx); } } From ebca69c9fb07dd7b0be7aa008215389581b193a0 Mon Sep 17 00:00:00 2001 From: Valerii Hiora Date: Fri, 12 Sep 2014 07:24:28 +0300 Subject: [PATCH 108/352] Fixed iOS build after OR1 changes --- config.sub | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config.sub b/config.sub index d654d03c..0ccff770 100755 --- a/config.sub +++ b/config.sub @@ -1404,6 +1404,9 @@ case $os in -mac*) os=`echo $os | sed -e 's|mac|macos|'` ;; + # Apple iOS + -ios*) + ;; -linux-dietlibc) os=-linux-dietlibc ;; From f1cf3ea4753260d37c9a43463bae2140e00e16f0 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Tue, 16 Sep 2014 04:42:33 -0400 Subject: [PATCH 109/352] fix tls_model autoconf test It has an unused variable, so it was always failing (at least with gcc 4.9.1). Alternatively, the `-Werror` flag could be removed if it isn't strictly necessary. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index d221876c..1d2e8902 100644 --- a/configure.ac +++ b/configure.ac @@ -411,7 +411,7 @@ SAVED_CFLAGS="${CFLAGS}" JE_CFLAGS_APPEND([-Werror]) JE_COMPILABLE([tls_model attribute], [], [static __thread int - __attribute__((tls_model("initial-exec"))) foo; + __attribute__((tls_model("initial-exec"), unused)) foo; foo = 0;], [je_cv_tls_model]) CFLAGS="${SAVED_CFLAGS}" From 913e9a8a853a693c5b5d6c13ab86f1b46a3404f7 Mon Sep 17 00:00:00 2001 From: Nick White Date: Fri, 19 Sep 2014 22:01:23 +0100 Subject: [PATCH 110/352] Generate a pkg-config file --- Makefile.in | 10 +++++++++- configure.ac | 3 +++ jemalloc.pc.in | 11 +++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 jemalloc.pc.in diff --git a/Makefile.in b/Makefile.in index ac56d8fa..41328b95 100644 --- a/Makefile.in +++ b/Makefile.in @@ -101,6 +101,7 @@ DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV) ifneq ($(SOREV),$(SO)) DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO) endif +PC := $(srcroot)jemalloc.pc MAN3 := $(objroot)doc/jemalloc$(install_suffix).3 DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) @@ -302,7 +303,14 @@ install_lib_static: $(STATIC_LIBS) install -m 755 $$l $(LIBDIR); \ done -install_lib: install_lib_shared install_lib_static +install_lib_pc: $(PC) + install -d $(LIBDIR)/pkgconfig + @for l in $(PC); do \ + echo "install -m 644 $$l $(LIBDIR)/pkgconfig"; \ + install -m 644 $$l $(LIBDIR)/pkgconfig; \ +done + +install_lib: install_lib_shared install_lib_static install_lib_pc install_doc_html: install -d $(DATADIR)/doc/jemalloc$(install_suffix) diff --git a/configure.ac b/configure.ac index 1d2e8902..2d5b56a4 100644 --- a/configure.ac +++ b/configure.ac @@ -540,6 +540,7 @@ je_="je_" AC_SUBST([je_]) cfgoutputs_in="Makefile.in" +cfgoutputs_in="${cfgoutputs_in} jemalloc.pc.in" cfgoutputs_in="${cfgoutputs_in} doc/html.xsl.in" cfgoutputs_in="${cfgoutputs_in} doc/manpages.xsl.in" cfgoutputs_in="${cfgoutputs_in} doc/jemalloc.xml.in" @@ -551,6 +552,7 @@ cfgoutputs_in="${cfgoutputs_in} test/test.sh.in" cfgoutputs_in="${cfgoutputs_in} test/include/test/jemalloc_test.h.in" cfgoutputs_out="Makefile" +cfgoutputs_out="${cfgoutputs_out} jemalloc.pc" cfgoutputs_out="${cfgoutputs_out} doc/html.xsl" cfgoutputs_out="${cfgoutputs_out} doc/manpages.xsl" cfgoutputs_out="${cfgoutputs_out} doc/jemalloc.xml" @@ -562,6 +564,7 @@ cfgoutputs_out="${cfgoutputs_out} test/test.sh" cfgoutputs_out="${cfgoutputs_out} test/include/test/jemalloc_test.h" cfgoutputs_tup="Makefile" +cfgoutputs_tup="${cfgoutputs_tup} jemalloc.pc:jemalloc.pc.in" cfgoutputs_tup="${cfgoutputs_tup} doc/html.xsl:doc/html.xsl.in" cfgoutputs_tup="${cfgoutputs_tup} doc/manpages.xsl:doc/manpages.xsl.in" cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc.xml:doc/jemalloc.xml.in" diff --git a/jemalloc.pc.in b/jemalloc.pc.in new file mode 100644 index 00000000..af3f945d --- /dev/null +++ b/jemalloc.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: jemalloc +Description: A general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support. +URL: http://www.canonware.com/jemalloc +Version: @jemalloc_version@ +Cflags: -I${includedir} +Libs: -L${libdir} -ljemalloc From 42f59559384ddb1af22607ddb3fe766b7b6ab0b7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 21 Sep 2014 21:40:38 -0700 Subject: [PATCH 111/352] Ignore jemalloc.pc . --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 79d454f2..fd68315d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,8 @@ /doc/jemalloc.html /doc/jemalloc.3 +/jemalloc.pc + /lib/ /Makefile From 5460aa6f6676c7f253bfcb75c028dfd38cae8aaf Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 22 Sep 2014 21:09:23 -0700 Subject: [PATCH 112/352] Convert all tsd variables to reside in a single tsd structure. --- include/jemalloc/internal/arena.h | 48 +-- include/jemalloc/internal/ckh.h | 8 +- include/jemalloc/internal/huge.h | 10 +- .../jemalloc/internal/jemalloc_internal.h.in | 168 ++++---- include/jemalloc/internal/private_symbols.txt | 96 ++--- include/jemalloc/internal/prof.h | 78 ++-- include/jemalloc/internal/quarantine.h | 21 +- include/jemalloc/internal/tcache.h | 88 ++--- include/jemalloc/internal/tsd.h | 341 ++++++++++------ src/arena.c | 23 +- src/ckh.c | 38 +- src/ctl.c | 93 ++++- src/huge.c | 21 +- src/jemalloc.c | 366 ++++++++++-------- src/prof.c | 244 ++++++------ src/quarantine.c | 99 ++--- src/rtree.c | 6 +- src/tcache.c | 101 ++--- src/tsd.c | 51 ++- test/unit/ckh.c | 46 ++- test/unit/rtree.c | 8 +- test/unit/tsd.c | 8 +- 22 files changed, 1027 insertions(+), 935 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index bfb0b3cf..f1a12057 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -419,9 +419,9 @@ extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; #endif bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); -void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, - bool try_tcache_dalloc); +void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, + size_t size, size_t extra, size_t alignment, bool zero, + bool try_tcache_alloc, bool try_tcache_dalloc); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, @@ -485,10 +485,12 @@ unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); prof_tctx_t *arena_prof_tctx_get(const void *ptr); void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); -void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); +void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, + bool try_tcache); size_t arena_salloc(const void *ptr, bool demote); -void arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache); -void arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, +void arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, + bool try_tcache); +void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache); #endif @@ -1053,7 +1055,8 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) } JEMALLOC_ALWAYS_INLINE void * -arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) +arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, + bool try_tcache) { tcache_t *tcache; @@ -1061,12 +1064,12 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) assert(size <= arena_maxclass); if (likely(size <= SMALL_MAXCLASS)) { - if (likely(try_tcache) && likely((tcache = tcache_get(true)) != - NULL)) + if (likely(try_tcache) && likely((tcache = tcache_get(tsd, + true)) != NULL)) return (tcache_alloc_small(tcache, size, zero)); else { - return (arena_malloc_small(choose_arena(arena), size, - zero)); + return (arena_malloc_small(choose_arena(tsd, arena), + size, zero)); } } else { /* @@ -1074,11 +1077,11 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) * infinite recursion during tcache initialization. */ if (try_tcache && size <= tcache_maxclass && likely((tcache = - tcache_get(true)) != NULL)) + tcache_get(tsd, true)) != NULL)) return (tcache_alloc_large(tcache, size, zero)); else { - return (arena_malloc_large(choose_arena(arena), size, - zero)); + return (arena_malloc_large(choose_arena(tsd, arena), + size, zero)); } } } @@ -1128,7 +1131,7 @@ arena_salloc(const void *ptr, bool demote) } JEMALLOC_ALWAYS_INLINE void -arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) +arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) { size_t pageind, mapbits; tcache_t *tcache; @@ -1141,8 +1144,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) assert(arena_mapbits_allocated_get(chunk, pageind) != 0); if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { /* Small allocation. */ - if (likely(try_tcache) && likely((tcache = tcache_get(false)) != - NULL)) { + if (likely(try_tcache) && likely((tcache = tcache_get(tsd, + false)) != NULL)) { size_t binind = arena_ptr_small_binind_get(ptr, mapbits); tcache_dalloc_small(tcache, ptr, binind); @@ -1154,7 +1157,7 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) assert(((uintptr_t)ptr & PAGE_MASK) == 0); if (try_tcache && size <= tcache_maxclass && likely((tcache = - tcache_get(false)) != NULL)) { + tcache_get(tsd, false)) != NULL)) { tcache_dalloc_large(tcache, ptr, size); } else arena_dalloc_large(chunk->arena, chunk, ptr); @@ -1162,7 +1165,8 @@ arena_dalloc(arena_chunk_t *chunk, void *ptr, bool try_tcache) } JEMALLOC_ALWAYS_INLINE void -arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache) +arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, + bool try_tcache) { tcache_t *tcache; @@ -1171,8 +1175,8 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache) if (likely(size <= SMALL_MAXCLASS)) { /* Small allocation. */ - if (likely(try_tcache) && likely((tcache = tcache_get(false)) != - NULL)) { + if (likely(try_tcache) && likely((tcache = tcache_get(tsd, + false)) != NULL)) { size_t binind = small_size2bin(size); tcache_dalloc_small(tcache, ptr, binind); } else { @@ -1184,7 +1188,7 @@ arena_sdalloc(arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache) assert(((uintptr_t)ptr & PAGE_MASK) == 0); if (try_tcache && size <= tcache_maxclass && (tcache = - tcache_get(false)) != NULL) { + tcache_get(tsd, false)) != NULL) { tcache_dalloc_large(tcache, ptr, size); } else arena_dalloc_large(chunk->arena, chunk, ptr); diff --git a/include/jemalloc/internal/ckh.h b/include/jemalloc/internal/ckh.h index 58712a6a..75c1c979 100644 --- a/include/jemalloc/internal/ckh.h +++ b/include/jemalloc/internal/ckh.h @@ -66,13 +66,13 @@ struct ckh_s { /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -bool ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, +bool ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp); -void ckh_delete(ckh_t *ckh); +void ckh_delete(tsd_t *tsd, ckh_t *ckh); size_t ckh_count(ckh_t *ckh); bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data); -bool ckh_insert(ckh_t *ckh, const void *key, const void *data); -bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key, +bool ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data); +bool ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, void **data); bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data); void ckh_string_hash(const void *key, size_t r_hash[2]); diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 2ec77520..b061e15b 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -9,12 +9,14 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -void *huge_malloc(arena_t *arena, size_t size, bool zero); -void *huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); +void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero); +void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, + bool zero); bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra); -void *huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc); +void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, + size_t size, size_t extra, size_t alignment, bool zero, + bool try_tcache_dalloc); #ifdef JEMALLOC_JET typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index a380a414..bff2bd27 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -350,7 +350,6 @@ static const bool config_ivsalloc = #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" @@ -364,15 +363,7 @@ static const bool config_ivsalloc = #include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" -typedef struct { - uint64_t allocated; - uint64_t deallocated; -} thread_allocated_t; -/* - * The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro - * argument. - */ -#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0}) +#include "jemalloc/internal/tsd.h" #undef JEMALLOC_H_STRUCTS /******************************************************************************/ @@ -407,8 +398,10 @@ extern unsigned narenas_total; extern unsigned narenas_auto; /* Read-only after initialization. */ arena_t *arenas_extend(unsigned ind); -void arenas_cleanup(void *arg); -arena_t *choose_arena_hard(void); +arena_t *choose_arena_hard(tsd_t *tsd); +void thread_allocated_cleanup(tsd_t *tsd); +void thread_deallocated_cleanup(tsd_t *tsd); +void arena_cleanup(tsd_t *tsd); void jemalloc_prefork(void); void jemalloc_postfork_parent(void); void jemalloc_postfork_child(void); @@ -422,7 +415,6 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" @@ -435,6 +427,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/hash.h" #include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" +#include "jemalloc/internal/tsd.h" #undef JEMALLOC_H_EXTERNS /******************************************************************************/ @@ -465,23 +458,13 @@ void jemalloc_postfork_child(void); #undef JEMALLOC_ARENA_INLINE_A #ifndef JEMALLOC_ENABLE_INLINE -malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *) - size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment); unsigned narenas_total_get(void); -arena_t *choose_arena(arena_t *arena); +arena_t *choose_arena(tsd_t *tsd, arena_t *arena); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -/* - * Map of pthread_self() --> arenas[???], used for selecting an arena to use - * for allocations. - */ -malloc_tsd_externs(arenas, arena_t *) -malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL, - arenas_cleanup) - /* * Compute usable size that would result from allocating an object with the * specified size. @@ -589,15 +572,15 @@ narenas_total_get(void) /* Choose an arena based on a per-thread value. */ JEMALLOC_INLINE arena_t * -choose_arena(arena_t *arena) +choose_arena(tsd_t *tsd, arena_t *arena) { arena_t *ret; if (arena != NULL) return (arena); - if ((ret = *arenas_tsd_get()) == NULL) { - ret = choose_arena_hard(); + if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) { + ret = choose_arena_hard(tsd); assert(ret != NULL); } @@ -622,72 +605,72 @@ choose_arena(arena_t *arena) #include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE -void *imalloct(size_t size, bool try_tcache, arena_t *arena); -void *imalloc(size_t size); -void *icalloct(size_t size, bool try_tcache, arena_t *arena); -void *icalloc(size_t size); -void *ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena); -void *ipalloc(size_t usize, size_t alignment, bool zero); +void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); +void *imalloc(tsd_t *tsd, size_t size); +void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); +void *icalloc(tsd_t *tsd, size_t size); +void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, + bool try_tcache, arena_t *arena); +void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); size_t isalloc(const void *ptr, bool demote); size_t ivsalloc(const void *ptr, bool demote); size_t u2rz(size_t usize); size_t p2rz(const void *ptr); -void idalloct(void *ptr, bool try_tcache); -void isdalloct(void *ptr, size_t size, bool try_tcache); -void idalloc(void *ptr); -void iqalloc(void *ptr, bool try_tcache); -void isqalloc(void *ptr, size_t size, bool try_tcache); -void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, - arena_t *arena); -void *iralloct(void *ptr, size_t size, size_t alignment, bool zero, - bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); -void *iralloc(void *ptr, size_t size, size_t alignment, bool zero); +void idalloct(tsd_t *tsd, void *ptr, bool try_tcache); +void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); +void idalloc(tsd_t *tsd, void *ptr); +void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache); +void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); +void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc, arena_t *arena); +void *iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, + bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); +void *iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, + bool zero); bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero); -malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) JEMALLOC_ALWAYS_INLINE void * -imalloct(size_t size, bool try_tcache, arena_t *arena) +imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) { assert(size != 0); if (size <= arena_maxclass) - return (arena_malloc(arena, size, false, try_tcache)); + return (arena_malloc(tsd, arena, size, false, try_tcache)); else - return (huge_malloc(arena, size, false)); + return (huge_malloc(tsd, arena, size, false)); } JEMALLOC_ALWAYS_INLINE void * -imalloc(size_t size) +imalloc(tsd_t *tsd, size_t size) { - return (imalloct(size, true, NULL)); + return (imalloct(tsd, size, true, NULL)); } JEMALLOC_ALWAYS_INLINE void * -icalloct(size_t size, bool try_tcache, arena_t *arena) +icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) { if (size <= arena_maxclass) - return (arena_malloc(arena, size, true, try_tcache)); + return (arena_malloc(tsd, arena, size, true, try_tcache)); else - return (huge_malloc(arena, size, true)); + return (huge_malloc(tsd, arena, size, true)); } JEMALLOC_ALWAYS_INLINE void * -icalloc(size_t size) +icalloc(tsd_t *tsd, size_t size) { - return (icalloct(size, true, NULL)); + return (icalloct(tsd, size, true, NULL)); } JEMALLOC_ALWAYS_INLINE void * -ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, +ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena) { void *ret; @@ -696,15 +679,15 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, assert(usize == sa2u(usize, alignment)); if (usize <= arena_maxclass && alignment <= PAGE) - ret = arena_malloc(arena, usize, zero, try_tcache); + ret = arena_malloc(tsd, arena, usize, zero, try_tcache); else { if (usize <= arena_maxclass) { - ret = arena_palloc(choose_arena(arena), usize, + ret = arena_palloc(choose_arena(tsd, arena), usize, alignment, zero); } else if (alignment <= chunksize) - ret = huge_malloc(arena, usize, zero); + ret = huge_malloc(tsd, arena, usize, zero); else - ret = huge_palloc(arena, usize, alignment, zero); + ret = huge_palloc(tsd, arena, usize, alignment, zero); } assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); @@ -712,10 +695,10 @@ ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, } JEMALLOC_ALWAYS_INLINE void * -ipalloc(size_t usize, size_t alignment, bool zero) +ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { - return (ipalloct(usize, alignment, zero, true, NULL)); + return (ipalloct(tsd, usize, alignment, zero, true, NULL)); } /* @@ -776,7 +759,7 @@ p2rz(const void *ptr) } JEMALLOC_ALWAYS_INLINE void -idalloct(void *ptr, bool try_tcache) +idalloct(tsd_t *tsd, void *ptr, bool try_tcache) { arena_chunk_t *chunk; @@ -784,13 +767,13 @@ idalloct(void *ptr, bool try_tcache) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(chunk, ptr, try_tcache); + arena_dalloc(tsd, chunk, ptr, try_tcache); else huge_dalloc(ptr); } JEMALLOC_ALWAYS_INLINE void -isdalloct(void *ptr, size_t size, bool try_tcache) +isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) { arena_chunk_t *chunk; @@ -798,42 +781,42 @@ isdalloct(void *ptr, size_t size, bool try_tcache) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_sdalloc(chunk, ptr, size, try_tcache); + arena_sdalloc(tsd, chunk, ptr, size, try_tcache); else huge_dalloc(ptr); } JEMALLOC_ALWAYS_INLINE void -idalloc(void *ptr) +idalloc(tsd_t *tsd, void *ptr) { - idalloct(ptr, true); + idalloct(tsd, ptr, true); } JEMALLOC_ALWAYS_INLINE void -iqalloc(void *ptr, bool try_tcache) +iqalloc(tsd_t *tsd, void *ptr, bool try_tcache) { if (config_fill && unlikely(opt_quarantine)) - quarantine(ptr); + quarantine(tsd, ptr); else - idalloct(ptr, try_tcache); + idalloct(tsd, ptr, try_tcache); } JEMALLOC_ALWAYS_INLINE void -isqalloc(void *ptr, size_t size, bool try_tcache) +isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) { if (config_fill && unlikely(opt_quarantine)) - quarantine(ptr); + quarantine(tsd, ptr); else - isdalloct(ptr, size, try_tcache); + isdalloct(tsd, ptr, size, try_tcache); } JEMALLOC_ALWAYS_INLINE void * -iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, - arena_t *arena) +iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc, arena_t *arena) { void *p; size_t usize, copysize; @@ -841,7 +824,7 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); + p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena); if (p == NULL) { if (extra == 0) return (NULL); @@ -849,7 +832,8 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, usize = sa2u(size, alignment); if (usize == 0) return (NULL); - p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); + p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, + arena); if (p == NULL) return (NULL); } @@ -859,12 +843,12 @@ iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - iqalloc(ptr, try_tcache_dalloc); + iqalloc(tsd, ptr, try_tcache_dalloc); return (p); } JEMALLOC_ALWAYS_INLINE void * -iralloct(void *ptr, size_t size, size_t alignment, bool zero, +iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) { size_t oldsize; @@ -880,24 +864,24 @@ iralloct(void *ptr, size_t size, size_t alignment, bool zero, * Existing object alignment is inadequate; allocate new space * and copy. */ - return (iralloct_realign(ptr, oldsize, size, 0, alignment, zero, - try_tcache_alloc, try_tcache_dalloc, arena)); + return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment, + zero, try_tcache_alloc, try_tcache_dalloc, arena)); } if (size <= arena_maxclass) { - return (arena_ralloc(arena, ptr, oldsize, size, 0, alignment, - zero, try_tcache_alloc, try_tcache_dalloc)); + return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, + alignment, zero, try_tcache_alloc, try_tcache_dalloc)); } else { - return (huge_ralloc(arena, ptr, oldsize, size, 0, alignment, - zero, try_tcache_dalloc)); + return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0, + alignment, zero, try_tcache_dalloc)); } } JEMALLOC_ALWAYS_INLINE void * -iralloc(void *ptr, size_t size, size_t alignment, bool zero) +iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero) { - return (iralloct(ptr, size, alignment, zero, true, true, NULL)); + return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL)); } JEMALLOC_ALWAYS_INLINE bool @@ -920,10 +904,6 @@ ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) else return (huge_ralloc_no_move(ptr, oldsize, size, extra)); } - -malloc_tsd_externs(thread_allocated, thread_allocated_t) -malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t, - THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup) #endif #include "jemalloc/internal/prof.h" diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index b8990177..84d48d19 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -8,6 +8,7 @@ arena_bitselm_get arena_boot arena_chunk_alloc_huge arena_chunk_dalloc_huge +arena_cleanup arena_dalloc arena_dalloc_bin arena_dalloc_bin_locked @@ -65,19 +66,9 @@ arena_sdalloc arena_stats_merge arena_tcache_fill_small arenas -arenas_booted arenas_cleanup arenas_extend -arenas_initialized arenas_lock -arenas_tls -arenas_tsd -arenas_tsd_boot -arenas_tsd_cleanup_wrapper -arenas_tsd_get -arenas_tsd_get_wrapper -arenas_tsd_init_head -arenas_tsd_set atomic_add_u atomic_add_uint32 atomic_add_uint64 @@ -317,37 +308,17 @@ prof_sample_accum_update prof_sample_threshold_update prof_tctx_get prof_tctx_set -prof_tdata_booted prof_tdata_cleanup prof_tdata_get prof_tdata_init -prof_tdata_initialized -prof_tdata_tls -prof_tdata_tsd -prof_tdata_tsd_boot -prof_tdata_tsd_cleanup_wrapper -prof_tdata_tsd_get -prof_tdata_tsd_get_wrapper -prof_tdata_tsd_init_head -prof_tdata_tsd_set prof_thread_active_get prof_thread_active_set prof_thread_name_get prof_thread_name_set quarantine quarantine_alloc_hook -quarantine_boot -quarantine_booted quarantine_cleanup quarantine_init -quarantine_tls -quarantine_tsd -quarantine_tsd_boot -quarantine_tsd_cleanup_wrapper -quarantine_tsd_get -quarantine_tsd_get_wrapper -quarantine_tsd_init_head -quarantine_tsd_set register_zone rtree_delete rtree_get @@ -386,55 +357,52 @@ tcache_arena_dissociate tcache_bin_flush_large tcache_bin_flush_small tcache_bin_info -tcache_boot0 -tcache_boot1 -tcache_booted +tcache_boot +tcache_cleanup tcache_create tcache_dalloc_large tcache_dalloc_small -tcache_destroy -tcache_enabled_booted +tcache_enabled_cleanup tcache_enabled_get -tcache_enabled_initialized tcache_enabled_set -tcache_enabled_tls -tcache_enabled_tsd -tcache_enabled_tsd_boot -tcache_enabled_tsd_cleanup_wrapper -tcache_enabled_tsd_get -tcache_enabled_tsd_get_wrapper -tcache_enabled_tsd_init_head -tcache_enabled_tsd_set tcache_event tcache_event_hard tcache_flush tcache_get tcache_get_hard -tcache_initialized tcache_maxclass tcache_salloc tcache_stats_merge -tcache_thread_cleanup -tcache_tls -tcache_tsd -tcache_tsd_boot -tcache_tsd_cleanup_wrapper -tcache_tsd_get -tcache_tsd_get_wrapper -tcache_tsd_init_head -tcache_tsd_set -thread_allocated_booted -thread_allocated_initialized -thread_allocated_tls -thread_allocated_tsd -thread_allocated_tsd_boot -thread_allocated_tsd_cleanup_wrapper -thread_allocated_tsd_get -thread_allocated_tsd_get_wrapper -thread_allocated_tsd_init_head -thread_allocated_tsd_set +thread_allocated_cleanup +thread_deallocated_cleanup +tsd_booted +tsd_arena_get +tsd_arena_set +tsd_boot +tsd_cleanup +tsd_cleanup_wrapper +tsd_get +tsd_get_wrapper +tsd_initialized tsd_init_check_recursion tsd_init_finish +tsd_init_head +tsd_quarantine_get +tsd_quarantine_set +tsd_set +tsd_tcache_enabled_get +tsd_tcache_enabled_set +tsd_tcache_get +tsd_tcache_set +tsd_tls +tsd_tsd +tsd_prof_tdata_get +tsd_prof_tdata_set +tsd_thread_allocated_get +tsd_thread_allocated_set +tsd_thread_deallocated_get +tsd_thread_deallocated_set +tsd_tryget u2rz valgrind_freelike_block valgrind_make_mem_defined diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index a1e7ac5e..b8a8b419 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -248,13 +248,13 @@ extern uint64_t prof_interval; */ extern size_t lg_prof_sample; -void prof_alloc_rollback(prof_tctx_t *tctx, bool updated); +void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx); -void prof_free_sampled_object(size_t usize, prof_tctx_t *tctx); +void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt); -prof_tctx_t *prof_lookup(prof_bt_t *bt); +prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); #ifdef JEMALLOC_JET size_t prof_bt_count(void); typedef int (prof_dump_open_t)(bool, const char *); @@ -263,12 +263,12 @@ extern prof_dump_open_t *prof_dump_open; void prof_idump(void); bool prof_mdump(const char *filename); void prof_gdump(void); -prof_tdata_t *prof_tdata_init(void); -prof_tdata_t *prof_tdata_reinit(prof_tdata_t *tdata); -void prof_reset(size_t lg_sample); -void prof_tdata_cleanup(void *arg); +prof_tdata_t *prof_tdata_init(tsd_t *tsd); +prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); +void prof_reset(tsd_t *tsd, size_t lg_sample); +void prof_tdata_cleanup(tsd_t *tsd); const char *prof_thread_name_get(void); -bool prof_thread_name_set(const char *thread_name); +bool prof_thread_name_set(tsd_t *tsd, const char *thread_name); bool prof_thread_active_get(void); bool prof_thread_active_set(bool active); void prof_boot0(void); @@ -284,43 +284,38 @@ void prof_sample_threshold_update(prof_tdata_t *tdata); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) - -prof_tdata_t *prof_tdata_get(bool create); -bool prof_sample_accum_update(size_t usize, bool commit, +prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); +bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, prof_tdata_t **tdata_out); -prof_tctx_t *prof_alloc_prep(size_t usize, bool update); +prof_tctx_t *prof_alloc_prep(tsd_t *tsd, size_t usize, bool update); prof_tctx_t *prof_tctx_get(const void *ptr); void prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx); void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx); -void prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, - bool updated, size_t old_usize, prof_tctx_t *old_tctx); -void prof_free(const void *ptr, size_t usize); +void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, + prof_tctx_t *tctx, bool updated, size_t old_usize, prof_tctx_t *old_tctx); +void prof_free(tsd_t *tsd, const void *ptr, size_t usize); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) -/* Thread-specific backtrace cache, used to reduce bt2gctx contention. */ -malloc_tsd_externs(prof_tdata, prof_tdata_t *) -malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, - prof_tdata_cleanup) - JEMALLOC_INLINE prof_tdata_t * -prof_tdata_get(bool create) +prof_tdata_get(tsd_t *tsd, bool create) { prof_tdata_t *tdata; cassert(config_prof); - tdata = *prof_tdata_tsd_get(); + tdata = tsd_prof_tdata_get(tsd); if (create) { - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { - if (tdata == NULL) - tdata = prof_tdata_init(); - } else if (tdata->state == prof_tdata_state_expired) - tdata = prof_tdata_reinit(tdata); - assert((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX || + if (unlikely(tdata == NULL)) { + tdata = prof_tdata_init(tsd); + tsd_prof_tdata_set(tsd, tdata); + } else if (unlikely(tdata->state == prof_tdata_state_expired)) { + tdata = prof_tdata_reinit(tsd, tdata); + tsd_prof_tdata_set(tsd, tdata); + } + assert(tdata == NULL || tdata->state == prof_tdata_state_attached); } @@ -363,13 +358,14 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx) } JEMALLOC_INLINE bool -prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out) +prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, + prof_tdata_t **tdata_out) { prof_tdata_t *tdata; cassert(config_prof); - tdata = prof_tdata_get(true); + tdata = prof_tdata_get(tsd, true); if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) tdata = NULL; @@ -392,7 +388,7 @@ prof_sample_accum_update(size_t usize, bool update, prof_tdata_t **tdata_out) } JEMALLOC_INLINE prof_tctx_t * -prof_alloc_prep(size_t usize, bool update) +prof_alloc_prep(tsd_t *tsd, size_t usize, bool update) { prof_tctx_t *ret; prof_tdata_t *tdata; @@ -400,13 +396,13 @@ prof_alloc_prep(size_t usize, bool update) assert(usize == s2u(usize)); - if (!opt_prof_active || likely(prof_sample_accum_update(usize, update, - &tdata))) + if (!opt_prof_active || likely(prof_sample_accum_update(tsd, usize, + update, &tdata))) ret = (prof_tctx_t *)(uintptr_t)1U; else { bt_init(&bt, tdata->vec); prof_backtrace(&bt); - ret = prof_lookup(&bt); + ret = prof_lookup(tsd, &bt); } return (ret); @@ -427,8 +423,8 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) } JEMALLOC_INLINE void -prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated, - size_t old_usize, prof_tctx_t *old_tctx) +prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, + bool updated, size_t old_usize, prof_tctx_t *old_tctx) { cassert(config_prof); @@ -436,7 +432,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated, if (!updated && ptr != NULL) { assert(usize == isalloc(ptr, true)); - if (prof_sample_accum_update(usize, true, NULL)) { + if (prof_sample_accum_update(tsd, usize, true, NULL)) { /* * Don't sample. The usize passed to PROF_ALLOC_PREP() * was larger than what actually got allocated, so a @@ -449,7 +445,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated, } if (unlikely((uintptr_t)old_tctx > (uintptr_t)1U)) - prof_free_sampled_object(old_usize, old_tctx); + prof_free_sampled_object(tsd, old_usize, old_tctx); if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) prof_malloc_sample_object(ptr, usize, tctx); else @@ -457,7 +453,7 @@ prof_realloc(const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated, } JEMALLOC_INLINE void -prof_free(const void *ptr, size_t usize) +prof_free(tsd_t *tsd, const void *ptr, size_t usize) { prof_tctx_t *tctx = prof_tctx_get(ptr); @@ -465,7 +461,7 @@ prof_free(const void *ptr, size_t usize) assert(usize == isalloc(ptr, true)); if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) - prof_free_sampled_object(usize, tctx); + prof_free_sampled_object(tsd, usize, tctx); } #endif diff --git a/include/jemalloc/internal/quarantine.h b/include/jemalloc/internal/quarantine.h index 16f677f7..3a755985 100644 --- a/include/jemalloc/internal/quarantine.h +++ b/include/jemalloc/internal/quarantine.h @@ -29,36 +29,29 @@ struct quarantine_s { /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -quarantine_t *quarantine_init(size_t lg_maxobjs); -void quarantine(void *ptr); -void quarantine_cleanup(void *arg); -bool quarantine_boot(void); +quarantine_t *quarantine_init(tsd_t *tsd, size_t lg_maxobjs); +void quarantine(tsd_t *tsd, void *ptr); +void quarantine_cleanup(tsd_t *tsd); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *) - void quarantine_alloc_hook(void); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_)) -malloc_tsd_externs(quarantine, quarantine_t *) -malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL, - quarantine_cleanup) - JEMALLOC_ALWAYS_INLINE void quarantine_alloc_hook(void) { - quarantine_t *quarantine; + tsd_t *tsd; assert(config_fill && opt_quarantine); - quarantine = *quarantine_tsd_get(); - if (quarantine == NULL) - quarantine_init(LG_MAXOBJS_INIT); + tsd = tsd_tryget(); + if (tsd != NULL && tsd_quarantine_get(tsd) == NULL) + tsd_quarantine_set(tsd, quarantine_init(tsd, LG_MAXOBJS_INIT)); } #endif diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index c9d723aa..6804668f 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -110,26 +110,22 @@ void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_associate(tcache_t *tcache, arena_t *arena); void tcache_arena_dissociate(tcache_t *tcache); -tcache_t *tcache_get_hard(tcache_t *tcache, bool create); +tcache_t *tcache_get_hard(tsd_t *tsd); tcache_t *tcache_create(arena_t *arena); -void tcache_destroy(tcache_t *tcache); -void tcache_thread_cleanup(void *arg); +void tcache_cleanup(tsd_t *tsd); +void tcache_enabled_cleanup(tsd_t *tsd); void tcache_stats_merge(tcache_t *tcache, arena_t *arena); -bool tcache_boot0(void); -bool tcache_boot1(void); +bool tcache_boot(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *) -malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t) - void tcache_event(tcache_t *tcache); void tcache_flush(void); bool tcache_enabled_get(void); -tcache_t *tcache_get(bool create); +tcache_t *tcache_get(tsd_t *tsd, bool create); void tcache_enabled_set(bool enabled); void *tcache_alloc_easy(tcache_bin_t *tbin); void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); @@ -139,41 +135,33 @@ void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) -/* Map of thread-specific caches. */ -malloc_tsd_externs(tcache, tcache_t *) -malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL, - tcache_thread_cleanup) -/* Per thread flag that allows thread caches to be disabled. */ -malloc_tsd_externs(tcache_enabled, tcache_enabled_t) -malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t, - tcache_enabled_default, malloc_tsd_no_cleanup) - JEMALLOC_INLINE void tcache_flush(void) { - tcache_t *tcache; + tsd_t *tsd; cassert(config_tcache); - tcache = *tcache_tsd_get(); - if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) - return; - tcache_destroy(tcache); - tcache = NULL; - tcache_tsd_set(&tcache); + tsd = tsd_tryget(); + if (tsd != NULL) + tcache_cleanup(tsd); } JEMALLOC_INLINE bool tcache_enabled_get(void) { + tsd_t *tsd; tcache_enabled_t tcache_enabled; cassert(config_tcache); - tcache_enabled = *tcache_enabled_tsd_get(); + tsd = tsd_tryget(); + if (tsd == NULL) + return (false); + tcache_enabled = tsd_tcache_enabled_get(tsd); if (tcache_enabled == tcache_enabled_default) { tcache_enabled = (tcache_enabled_t)opt_tcache; - tcache_enabled_tsd_set(&tcache_enabled); + tsd_tcache_enabled_set(tsd, tcache_enabled); } return ((bool)tcache_enabled); @@ -182,33 +170,24 @@ tcache_enabled_get(void) JEMALLOC_INLINE void tcache_enabled_set(bool enabled) { + tsd_t *tsd; tcache_enabled_t tcache_enabled; - tcache_t *tcache; cassert(config_tcache); + tsd = tsd_tryget(); + if (tsd == NULL) + return; + tcache_enabled = (tcache_enabled_t)enabled; - tcache_enabled_tsd_set(&tcache_enabled); - tcache = *tcache_tsd_get(); - if (enabled) { - if (tcache == TCACHE_STATE_DISABLED) { - tcache = NULL; - tcache_tsd_set(&tcache); - } - } else /* disabled */ { - if (tcache > TCACHE_STATE_MAX) { - tcache_destroy(tcache); - tcache = NULL; - } - if (tcache == NULL) { - tcache = TCACHE_STATE_DISABLED; - tcache_tsd_set(&tcache); - } - } + tsd_tcache_enabled_set(tsd, tcache_enabled); + + if (!enabled) + tcache_cleanup(tsd); } JEMALLOC_ALWAYS_INLINE tcache_t * -tcache_get(bool create) +tcache_get(tsd_t *tsd, bool create) { tcache_t *tcache; @@ -216,12 +195,19 @@ tcache_get(bool create) return (NULL); if (config_lazy_lock && isthreaded == false) return (NULL); + /* + * If create is true, the caller has already assured that tsd is + * non-NULL. + */ + if (!create && unlikely(tsd == NULL)) + return (NULL); - tcache = *tcache_tsd_get(); - if (unlikely((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX)) { - if (tcache == TCACHE_STATE_DISABLED) - return (NULL); - tcache = tcache_get_hard(tcache, create); + tcache = tsd_tcache_get(tsd); + if (!create) + return (tcache); + if (unlikely(tcache == NULL)) { + tcache = tcache_get_hard(tsd); + tsd_tcache_set(tsd, tcache); } return (tcache); diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 9fb4a23e..44952eed 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -12,6 +12,15 @@ typedef struct tsd_init_block_s tsd_init_block_t; typedef struct tsd_init_head_s tsd_init_head_t; #endif +typedef struct tsd_s tsd_t; + +typedef enum { + tsd_state_uninitialized, + tsd_state_nominal, + tsd_state_purgatory, + tsd_state_reincarnated +} tsd_state_t; + /* * TLS/TSD-agnostic macro-based implementation of thread-specific data. There * are four macros that support (at least) three use cases: file-private, @@ -24,11 +33,11 @@ typedef struct tsd_init_head_s tsd_init_head_t; * int y; * } example_t; * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) - * malloc_tsd_protos(, example, example_t *) - * malloc_tsd_externs(example, example_t *) + * malloc_tsd_protos(, example_, example_t *) + * malloc_tsd_externs(example_, example_t *) * In example.c: - * malloc_tsd_data(, example, example_t *, EX_INITIALIZER) - * malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER, + * malloc_tsd_data(, example_, example_t *, EX_INITIALIZER) + * malloc_tsd_funcs(, example_, example_t *, EX_INITIALIZER, * example_tsd_cleanup) * * The result is a set of generated functions, e.g.: @@ -43,15 +52,13 @@ typedef struct tsd_init_head_s tsd_init_head_t; * cast to (void *). This means that the cleanup function needs to cast *and* * dereference the function argument, e.g.: * - * void + * bool * example_tsd_cleanup(void *arg) * { * example_t *example = *(example_t **)arg; * * [...] - * if ([want the cleanup function to be called again]) { - * example_tsd_set(&example); - * } + * return ([want the cleanup function to be called again]); * } * * If example_tsd_set() is called within example_tsd_cleanup(), it will be @@ -63,60 +70,60 @@ typedef struct tsd_init_head_s tsd_init_head_t; /* malloc_tsd_protos(). */ #define malloc_tsd_protos(a_attr, a_name, a_type) \ a_attr bool \ -a_name##_tsd_boot(void); \ +a_name##tsd_boot(void); \ a_attr a_type * \ -a_name##_tsd_get(void); \ +a_name##tsd_get(void); \ a_attr void \ -a_name##_tsd_set(a_type *val); +a_name##tsd_set(a_type *val); /* malloc_tsd_externs(). */ #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP #define malloc_tsd_externs(a_name, a_type) \ -extern __thread a_type a_name##_tls; \ -extern __thread bool a_name##_initialized; \ -extern bool a_name##_booted; +extern __thread a_type a_name##tsd_tls; \ +extern __thread bool a_name##tsd_initialized; \ +extern bool a_name##tsd_booted; #elif (defined(JEMALLOC_TLS)) #define malloc_tsd_externs(a_name, a_type) \ -extern __thread a_type a_name##_tls; \ -extern pthread_key_t a_name##_tsd; \ -extern bool a_name##_booted; +extern __thread a_type a_name##tsd_tls; \ +extern pthread_key_t a_name##tsd_tsd; \ +extern bool a_name##tsd_booted; #elif (defined(_WIN32)) #define malloc_tsd_externs(a_name, a_type) \ -extern DWORD a_name##_tsd; \ -extern bool a_name##_booted; +extern DWORD a_name##tsd_tsd; \ +extern bool a_name##tsd_booted; #else #define malloc_tsd_externs(a_name, a_type) \ -extern pthread_key_t a_name##_tsd; \ -extern tsd_init_head_t a_name##_tsd_init_head; \ -extern bool a_name##_booted; +extern pthread_key_t a_name##tsd_tsd; \ +extern tsd_init_head_t a_name##tsd_init_head; \ +extern bool a_name##tsd_booted; #endif /* malloc_tsd_data(). */ #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ a_attr __thread a_type JEMALLOC_TLS_MODEL \ - a_name##_tls = a_initializer; \ + a_name##tsd_tls = a_initializer; \ a_attr __thread bool JEMALLOC_TLS_MODEL \ - a_name##_initialized = false; \ -a_attr bool a_name##_booted = false; + a_name##tsd_initialized = false; \ +a_attr bool a_name##tsd_booted = false; #elif (defined(JEMALLOC_TLS)) #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ a_attr __thread a_type JEMALLOC_TLS_MODEL \ - a_name##_tls = a_initializer; \ -a_attr pthread_key_t a_name##_tsd; \ -a_attr bool a_name##_booted = false; + a_name##tsd_tls = a_initializer; \ +a_attr pthread_key_t a_name##tsd_tsd; \ +a_attr bool a_name##tsd_booted = false; #elif (defined(_WIN32)) #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ -a_attr DWORD a_name##_tsd; \ -a_attr bool a_name##_booted = false; +a_attr DWORD a_name##tsd_tsd; \ +a_attr bool a_name##tsd_booted = false; #else #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ -a_attr pthread_key_t a_name##_tsd; \ -a_attr tsd_init_head_t a_name##_tsd_init_head = { \ +a_attr pthread_key_t a_name##tsd_tsd; \ +a_attr tsd_init_head_t a_name##tsd_init_head = { \ ql_head_initializer(blocks), \ MALLOC_MUTEX_INITIALIZER \ }; \ -a_attr bool a_name##_booted = false; +a_attr bool a_name##tsd_booted = false; #endif /* malloc_tsd_funcs(). */ @@ -125,75 +132,76 @@ a_attr bool a_name##_booted = false; a_cleanup) \ /* Initialization/cleanup. */ \ a_attr bool \ -a_name##_tsd_cleanup_wrapper(void) \ +a_name##tsd_cleanup_wrapper(void) \ { \ \ - if (a_name##_initialized) { \ - a_name##_initialized = false; \ - a_cleanup(&a_name##_tls); \ + if (a_name##tsd_initialized) { \ + a_name##tsd_initialized = false; \ + a_cleanup(&a_name##tsd_tls); \ } \ - return (a_name##_initialized); \ + return (a_name##tsd_initialized); \ } \ a_attr bool \ -a_name##_tsd_boot(void) \ +a_name##tsd_boot(void) \ { \ \ if (a_cleanup != malloc_tsd_no_cleanup) { \ malloc_tsd_cleanup_register( \ - &a_name##_tsd_cleanup_wrapper); \ + &a_name##tsd_cleanup_wrapper); \ } \ - a_name##_booted = true; \ + a_name##tsd_booted = true; \ return (false); \ } \ /* Get/set. */ \ a_attr a_type * \ -a_name##_tsd_get(void) \ +a_name##tsd_get(void) \ { \ \ - assert(a_name##_booted); \ - return (&a_name##_tls); \ + assert(a_name##tsd_booted); \ + return (&a_name##tsd_tls); \ } \ a_attr void \ -a_name##_tsd_set(a_type *val) \ +a_name##tsd_set(a_type *val) \ { \ \ - assert(a_name##_booted); \ - a_name##_tls = (*val); \ + assert(a_name##tsd_booted); \ + a_name##tsd_tls = (*val); \ if (a_cleanup != malloc_tsd_no_cleanup) \ - a_name##_initialized = true; \ + a_name##tsd_initialized = true; \ } #elif (defined(JEMALLOC_TLS)) #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ a_cleanup) \ /* Initialization/cleanup. */ \ a_attr bool \ -a_name##_tsd_boot(void) \ +a_name##tsd_boot(void) \ { \ \ if (a_cleanup != malloc_tsd_no_cleanup) { \ - if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0) \ + if (pthread_key_create(&a_name##tsd_tsd, a_cleanup) != \ + 0) \ return (true); \ } \ - a_name##_booted = true; \ + a_name##tsd_booted = true; \ return (false); \ } \ /* Get/set. */ \ a_attr a_type * \ -a_name##_tsd_get(void) \ +a_name##tsd_get(void) \ { \ \ - assert(a_name##_booted); \ - return (&a_name##_tls); \ + assert(a_name##tsd_booted); \ + return (&a_name##tsd_tls); \ } \ a_attr void \ -a_name##_tsd_set(a_type *val) \ +a_name##tsd_set(a_type *val) \ { \ \ - assert(a_name##_booted); \ - a_name##_tls = (*val); \ + assert(a_name##tsd_booted); \ + a_name##tsd_tls = (*val); \ if (a_cleanup != malloc_tsd_no_cleanup) { \ - if (pthread_setspecific(a_name##_tsd, \ - (void *)(&a_name##_tls))) { \ + if (pthread_setspecific(a_name##tsd_tsd, \ + (void *)(&a_name##tsd_tls))) { \ malloc_write(": Error" \ " setting TSD for "#a_name"\n"); \ if (opt_abort) \ @@ -208,23 +216,20 @@ a_name##_tsd_set(a_type *val) \ typedef struct { \ bool initialized; \ a_type val; \ -} a_name##_tsd_wrapper_t; \ +} a_name##tsd_wrapper_t; \ /* Initialization/cleanup. */ \ a_attr bool \ -a_name##_tsd_cleanup_wrapper(void) \ +a_name##tsd_cleanup_wrapper(void) \ { \ - a_name##_tsd_wrapper_t *wrapper; \ + a_name##tsd_wrapper_t *wrapper; \ \ - wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \ + wrapper = (a_name##tsd_wrapper_t *)TlsGetValue(a_name##tsd_tsd);\ if (wrapper == NULL) \ return (false); \ if (a_cleanup != malloc_tsd_no_cleanup && \ wrapper->initialized) { \ - a_type val = wrapper->val; \ - a_type tsd_static_data = a_initializer; \ wrapper->initialized = false; \ - wrapper->val = tsd_static_data; \ - a_cleanup(&val); \ + a_cleanup(&wrapper->val); \ if (wrapper->initialized) { \ /* Trigger another cleanup round. */ \ return (true); \ @@ -234,39 +239,38 @@ a_name##_tsd_cleanup_wrapper(void) \ return (false); \ } \ a_attr bool \ -a_name##_tsd_boot(void) \ +a_name##tsd_boot(void) \ { \ \ - a_name##_tsd = TlsAlloc(); \ - if (a_name##_tsd == TLS_OUT_OF_INDEXES) \ + a_name##tsd_tsd = TlsAlloc(); \ + if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ return (true); \ if (a_cleanup != malloc_tsd_no_cleanup) { \ malloc_tsd_cleanup_register( \ - &a_name##_tsd_cleanup_wrapper); \ + &a_name##tsd_cleanup_wrapper); \ } \ - a_name##_booted = true; \ + a_name##tsd_booted = true; \ return (false); \ } \ /* Get/set. */ \ -a_attr a_name##_tsd_wrapper_t * \ -a_name##_tsd_get_wrapper(void) \ +a_attr a_name##tsd_wrapper_t * \ +a_name##tsd_get_wrapper(void) \ { \ - a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ - TlsGetValue(a_name##_tsd); \ + a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ + TlsGetValue(a_name##tsd_tsd); \ \ - if (wrapper == NULL) { \ - wrapper = (a_name##_tsd_wrapper_t *) \ - malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ + if (unlikely(wrapper == NULL)) { \ + wrapper = (a_name##tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ if (wrapper == NULL) { \ malloc_write(": Error allocating" \ " TSD for "#a_name"\n"); \ abort(); \ } else { \ - static a_type tsd_static_data = a_initializer; \ wrapper->initialized = false; \ - wrapper->val = tsd_static_data; \ + wrapper->val = a_initializer; \ } \ - if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \ + if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ malloc_write(": Error setting" \ " TSD for "#a_name"\n"); \ abort(); \ @@ -275,21 +279,21 @@ a_name##_tsd_get_wrapper(void) \ return (wrapper); \ } \ a_attr a_type * \ -a_name##_tsd_get(void) \ +a_name##tsd_get(void) \ { \ - a_name##_tsd_wrapper_t *wrapper; \ + a_name##tsd_wrapper_t *wrapper; \ \ - assert(a_name##_booted); \ - wrapper = a_name##_tsd_get_wrapper(); \ + assert(a_name##tsd_booted); \ + wrapper = a_name##tsd_get_wrapper(); \ return (&wrapper->val); \ } \ a_attr void \ -a_name##_tsd_set(a_type *val) \ +a_name##tsd_set(a_type *val) \ { \ - a_name##_tsd_wrapper_t *wrapper; \ + a_name##tsd_wrapper_t *wrapper; \ \ - assert(a_name##_booted); \ - wrapper = a_name##_tsd_get_wrapper(); \ + assert(a_name##tsd_booted); \ + wrapper = a_name##tsd_get_wrapper(); \ wrapper->val = *(val); \ if (a_cleanup != malloc_tsd_no_cleanup) \ wrapper->initialized = true; \ @@ -301,12 +305,12 @@ a_name##_tsd_set(a_type *val) \ typedef struct { \ bool initialized; \ a_type val; \ -} a_name##_tsd_wrapper_t; \ +} a_name##tsd_wrapper_t; \ /* Initialization/cleanup. */ \ a_attr void \ -a_name##_tsd_cleanup_wrapper(void *arg) \ +a_name##tsd_cleanup_wrapper(void *arg) \ { \ - a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\ + a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *)arg; \ \ if (a_cleanup != malloc_tsd_no_cleanup && \ wrapper->initialized) { \ @@ -314,7 +318,7 @@ a_name##_tsd_cleanup_wrapper(void *arg) \ a_cleanup(&wrapper->val); \ if (wrapper->initialized) { \ /* Trigger another cleanup round. */ \ - if (pthread_setspecific(a_name##_tsd, \ + if (pthread_setspecific(a_name##tsd_tsd, \ (void *)wrapper)) { \ malloc_write(": Error" \ " setting TSD for "#a_name"\n"); \ @@ -327,66 +331,65 @@ a_name##_tsd_cleanup_wrapper(void *arg) \ malloc_tsd_dalloc(wrapper); \ } \ a_attr bool \ -a_name##_tsd_boot(void) \ +a_name##tsd_boot(void) \ { \ \ - if (pthread_key_create(&a_name##_tsd, \ - a_name##_tsd_cleanup_wrapper) != 0) \ + if (pthread_key_create(&a_name##tsd_tsd, \ + a_name##tsd_cleanup_wrapper) != 0) \ return (true); \ - a_name##_booted = true; \ + a_name##tsd_booted = true; \ return (false); \ } \ /* Get/set. */ \ -a_attr a_name##_tsd_wrapper_t * \ -a_name##_tsd_get_wrapper(void) \ +a_attr a_name##tsd_wrapper_t * \ +a_name##tsd_get_wrapper(void) \ { \ - a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ - pthread_getspecific(a_name##_tsd); \ + a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ + pthread_getspecific(a_name##tsd_tsd); \ \ - if (wrapper == NULL) { \ + if (unlikely(wrapper == NULL)) { \ tsd_init_block_t block; \ wrapper = tsd_init_check_recursion( \ - &a_name##_tsd_init_head, &block); \ + &a_name##tsd_init_head, &block); \ if (wrapper) \ return (wrapper); \ - wrapper = (a_name##_tsd_wrapper_t *) \ - malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ + wrapper = (a_name##tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ block.data = wrapper; \ if (wrapper == NULL) { \ malloc_write(": Error allocating" \ " TSD for "#a_name"\n"); \ abort(); \ } else { \ - static a_type tsd_static_data = a_initializer; \ wrapper->initialized = false; \ - wrapper->val = tsd_static_data; \ + wrapper->val = a_initializer; \ } \ - if (pthread_setspecific(a_name##_tsd, \ + if (pthread_setspecific(a_name##tsd_tsd, \ (void *)wrapper)) { \ malloc_write(": Error setting" \ " TSD for "#a_name"\n"); \ abort(); \ } \ - tsd_init_finish(&a_name##_tsd_init_head, &block); \ + tsd_init_finish(&a_name##tsd_init_head, &block); \ } \ return (wrapper); \ } \ a_attr a_type * \ -a_name##_tsd_get(void) \ +a_name##tsd_get(void) \ { \ - a_name##_tsd_wrapper_t *wrapper; \ + a_name##tsd_wrapper_t *wrapper; \ \ - assert(a_name##_booted); \ - wrapper = a_name##_tsd_get_wrapper(); \ + assert(a_name##tsd_booted); \ + wrapper = a_name##tsd_get_wrapper(); \ return (&wrapper->val); \ } \ a_attr void \ -a_name##_tsd_set(a_type *val) \ +a_name##tsd_set(a_type *val) \ { \ - a_name##_tsd_wrapper_t *wrapper; \ + a_name##tsd_wrapper_t *wrapper; \ \ - assert(a_name##_booted); \ - wrapper = a_name##_tsd_get_wrapper(); \ + assert(a_name##tsd_booted); \ + wrapper = a_name##tsd_get_wrapper(); \ wrapper->val = *(val); \ if (a_cleanup != malloc_tsd_no_cleanup) \ wrapper->initialized = true; \ @@ -410,25 +413,123 @@ struct tsd_init_head_s { }; #endif +#define MALLOC_TSD \ +/* O(name, type) */ \ + O(tcache, tcache_t *) \ + O(thread_allocated, uint64_t) \ + O(thread_deallocated, uint64_t) \ + O(prof_tdata, prof_tdata_t *) \ + O(arena, arena_t *) \ + O(tcache_enabled, tcache_enabled_t) \ + O(quarantine, quarantine_t *) \ + +#define TSD_INITIALIZER { \ + tsd_state_uninitialized, \ + NULL, \ + 0, \ + 0, \ + NULL, \ + NULL, \ + tcache_enabled_default, \ + NULL \ +} + +struct tsd_s { + tsd_state_t state; +#define O(n, t) \ + t n; +MALLOC_TSD +#undef O +}; + +static const tsd_t tsd_initializer = TSD_INITIALIZER; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS void *malloc_tsd_malloc(size_t size); void malloc_tsd_dalloc(void *wrapper); -void malloc_tsd_no_cleanup(void *); +void malloc_tsd_no_cleanup(void *arg); void malloc_tsd_cleanup_register(bool (*f)(void)); -void malloc_tsd_boot(void); +bool malloc_tsd_boot(void); #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ !defined(_WIN32)) void *tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block); void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); #endif +void tsd_cleanup(void *arg); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES +#ifndef JEMALLOC_ENABLE_INLINE +malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) + +tsd_t *tsd_tryget(void); +#define O(n, t) \ +t *tsd_##n##p_get(tsd_t *tsd); \ +t tsd_##n##_get(tsd_t *tsd); \ +void tsd_##n##_set(tsd_t *tsd, t n); +MALLOC_TSD +#undef O +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TSD_C_)) +malloc_tsd_externs(, tsd_t) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) + +JEMALLOC_INLINE tsd_t * +tsd_tryget(void) +{ + tsd_t *tsd; + + tsd = tsd_get(); + if (unlikely(tsd == NULL)) + return (NULL); + + if (likely(tsd->state == tsd_state_nominal)) + return (tsd); + else if (tsd->state == tsd_state_uninitialized) { + tsd->state = tsd_state_nominal; + tsd_set(tsd); + return (tsd); + } else if (tsd->state == tsd_state_purgatory) { + tsd->state = tsd_state_reincarnated; + tsd_set(tsd); + return (NULL); + } else { + assert(tsd->state == tsd_state_reincarnated); + return (NULL); + } +} + +#define O(n, t) \ +JEMALLOC_INLINE t * \ +tsd_##n##p_get(tsd_t *tsd) \ +{ \ + \ + return (&tsd->n); \ +} \ + \ +JEMALLOC_INLINE t \ +tsd_##n##_get(tsd_t *tsd) \ +{ \ + \ + return (*tsd_##n##p_get(tsd)); \ +} \ + \ +JEMALLOC_INLINE void \ +tsd_##n##_set(tsd_t *tsd, t n) \ +{ \ + \ + tsd->n = n; \ +} +MALLOC_TSD +#undef O +#endif + #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/src/arena.c b/src/arena.c index 35d792a2..40da9f47 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2058,7 +2058,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, } void * -arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, +arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc) { @@ -2078,9 +2078,12 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); - } else - ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); + ret = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, + arena); + } else { + ret = arena_malloc(tsd, arena, size + extra, zero, + try_tcache_alloc); + } if (ret == NULL) { if (extra == 0) @@ -2090,10 +2093,12 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t usize = sa2u(size, alignment); if (usize == 0) return (NULL); - ret = ipalloct(usize, alignment, zero, try_tcache_alloc, - arena); - } else - ret = arena_malloc(arena, size, zero, try_tcache_alloc); + ret = ipalloct(tsd, usize, alignment, zero, + try_tcache_alloc, arena); + } else { + ret = arena_malloc(tsd, arena, size, zero, + try_tcache_alloc); + } if (ret == NULL) return (NULL); @@ -2108,7 +2113,7 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, copysize = (size < oldsize) ? size : oldsize; JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); - iqalloc(ptr, try_tcache_dalloc); + iqalloc(tsd, ptr, try_tcache_dalloc); return (ret); } diff --git a/src/ckh.c b/src/ckh.c index 04c52966..7c7cc098 100644 --- a/src/ckh.c +++ b/src/ckh.c @@ -40,8 +40,8 @@ /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static bool ckh_grow(ckh_t *ckh); -static void ckh_shrink(ckh_t *ckh); +static bool ckh_grow(tsd_t *tsd, ckh_t *ckh); +static void ckh_shrink(tsd_t *tsd, ckh_t *ckh); /******************************************************************************/ @@ -243,7 +243,7 @@ ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) } static bool -ckh_grow(ckh_t *ckh) +ckh_grow(tsd_t *tsd, ckh_t *ckh) { bool ret; ckhc_t *tab, *ttab; @@ -270,7 +270,7 @@ ckh_grow(ckh_t *ckh) ret = true; goto label_return; } - tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); + tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true); if (tab == NULL) { ret = true; goto label_return; @@ -282,12 +282,12 @@ ckh_grow(ckh_t *ckh) ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; if (ckh_rebuild(ckh, tab) == false) { - idalloc(tab); + idalloc(tsd, tab); break; } /* Rebuilding failed, so back out partially rebuilt table. */ - idalloc(ckh->tab); + idalloc(tsd, ckh->tab); ckh->tab = tab; ckh->lg_curbuckets = lg_prevbuckets; } @@ -298,7 +298,7 @@ label_return: } static void -ckh_shrink(ckh_t *ckh) +ckh_shrink(tsd_t *tsd, ckh_t *ckh) { ckhc_t *tab, *ttab; size_t lg_curcells, usize; @@ -313,7 +313,7 @@ ckh_shrink(ckh_t *ckh) usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); if (usize == 0) return; - tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); + tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true); if (tab == NULL) { /* * An OOM error isn't worth propagating, since it doesn't @@ -328,7 +328,7 @@ ckh_shrink(ckh_t *ckh) ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; if (ckh_rebuild(ckh, tab) == false) { - idalloc(tab); + idalloc(tsd, tab); #ifdef CKH_COUNT ckh->nshrinks++; #endif @@ -336,7 +336,7 @@ ckh_shrink(ckh_t *ckh) } /* Rebuilding failed, so back out partially rebuilt table. */ - idalloc(ckh->tab); + idalloc(tsd, ckh->tab); ckh->tab = tab; ckh->lg_curbuckets = lg_prevbuckets; #ifdef CKH_COUNT @@ -345,7 +345,8 @@ ckh_shrink(ckh_t *ckh) } bool -ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) +ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, + ckh_keycomp_t *keycomp) { bool ret; size_t mincells, usize; @@ -388,7 +389,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) ret = true; goto label_return; } - ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); + ckh->tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true); if (ckh->tab == NULL) { ret = true; goto label_return; @@ -400,7 +401,7 @@ label_return: } void -ckh_delete(ckh_t *ckh) +ckh_delete(tsd_t *tsd, ckh_t *ckh) { assert(ckh != NULL); @@ -417,7 +418,7 @@ ckh_delete(ckh_t *ckh) (unsigned long long)ckh->nrelocs); #endif - idalloc(ckh->tab); + idalloc(tsd, ckh->tab); if (config_debug) memset(ckh, 0x5a, sizeof(ckh_t)); } @@ -452,7 +453,7 @@ ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) } bool -ckh_insert(ckh_t *ckh, const void *key, const void *data) +ckh_insert(tsd_t *tsd, ckh_t *ckh, const void *key, const void *data) { bool ret; @@ -464,7 +465,7 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data) #endif while (ckh_try_insert(ckh, &key, &data)) { - if (ckh_grow(ckh)) { + if (ckh_grow(tsd, ckh)) { ret = true; goto label_return; } @@ -476,7 +477,8 @@ label_return: } bool -ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data) +ckh_remove(tsd_t *tsd, ckh_t *ckh, const void *searchkey, void **key, + void **data) { size_t cell; @@ -497,7 +499,7 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data) + LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets > ckh->lg_minbuckets) { /* Ignore error due to OOM. */ - ckh_shrink(ckh); + ckh_shrink(tsd, ckh); } return (false); diff --git a/src/ctl.c b/src/ctl.c index b816c845..c55f6e44 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -565,18 +565,23 @@ ctl_arena_refresh(arena_t *arena, unsigned i) static bool ctl_grow(void) { + tsd_t *tsd; ctl_arena_stats_t *astats; arena_t **tarenas; + tsd = tsd_tryget(); + if (tsd == NULL) + return (true); + /* Allocate extended arena stats and arenas arrays. */ - astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) * + astats = (ctl_arena_stats_t *)imalloc(tsd, (ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t)); if (astats == NULL) return (true); - tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) * + tarenas = (arena_t **)imalloc(tsd, (ctl_stats.narenas + 1) * sizeof(arena_t *)); if (tarenas == NULL) { - idalloc(astats); + idalloc(tsd, astats); return (true); } @@ -585,8 +590,8 @@ ctl_grow(void) sizeof(ctl_arena_stats_t)); memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) { - idalloc(tarenas); - idalloc(astats); + idalloc(tsd, tarenas); + idalloc(tsd, astats); return (true); } /* Swap merged stats to their new location. */ @@ -623,7 +628,7 @@ ctl_grow(void) * base_alloc()). */ if (ctl_stats.narenas != narenas_auto) - idalloc(arenas_old); + idalloc(tsd, arenas_old); } ctl_stats.arenas = astats; ctl_stats.narenas++; @@ -1105,6 +1110,31 @@ label_return: \ return (ret); \ } +#define CTL_TSD_RO_NL_CGEN(c, n, m, t) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + t oldval; \ + tsd_t *tsd; \ + \ + if ((c) == false) \ + return (ENOENT); \ + READONLY(); \ + tsd = tsd_tryget(); \ + if (tsd == NULL) { \ + ret = EAGAIN; \ + goto label_return; \ + } \ + oldval = (m(tsd)); \ + READ(oldval, t); \ + \ + ret = 0; \ +label_return: \ + return (ret); \ +} + #define CTL_RO_BOOL_CONFIG_GEN(n) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ @@ -1194,10 +1224,15 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; + tsd_t *tsd; unsigned newind, oldind; + tsd = tsd_tryget(); + if (tsd == NULL) + return (EAGAIN); + malloc_mutex_lock(&ctl_mtx); - newind = oldind = choose_arena(NULL)->ind; + newind = oldind = choose_arena(tsd, NULL)->ind; WRITE(newind, unsigned); READ(oldind, unsigned); if (newind != oldind) { @@ -1224,14 +1259,14 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, /* Set new arena association. */ if (config_tcache) { - tcache_t *tcache; - if ((uintptr_t)(tcache = *tcache_tsd_get()) > - (uintptr_t)TCACHE_STATE_MAX) { + tcache_t *tcache = tsd_tcache_get(tsd); + if (tcache != NULL) { tcache_arena_dissociate(tcache); tcache_arena_associate(tcache, arena); } } - arenas_tsd_set(&arena); + + tsd_arena_set(tsd, arena); } ret = 0; @@ -1240,14 +1275,14 @@ label_return: return (ret); } -CTL_RO_NL_CGEN(config_stats, thread_allocated, - thread_allocated_tsd_get()->allocated, uint64_t) -CTL_RO_NL_CGEN(config_stats, thread_allocatedp, - &thread_allocated_tsd_get()->allocated, uint64_t *) -CTL_RO_NL_CGEN(config_stats, thread_deallocated, - thread_allocated_tsd_get()->deallocated, uint64_t) -CTL_RO_NL_CGEN(config_stats, thread_deallocatedp, - &thread_allocated_tsd_get()->deallocated, uint64_t *) +CTL_TSD_RO_NL_CGEN(config_stats, thread_allocated, tsd_thread_allocated_get, + uint64_t) +CTL_TSD_RO_NL_CGEN(config_stats, thread_allocatedp, tsd_thread_allocatedp_get, + uint64_t *) +CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocated, tsd_thread_deallocated_get, + uint64_t) +CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, + tsd_thread_deallocatedp_get, uint64_t *) static int thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, @@ -1305,11 +1340,20 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp, oldname = prof_thread_name_get(); if (newp != NULL) { + tsd_t *tsd; + if (newlen != sizeof(const char *)) { ret = EINVAL; goto label_return; } - if (prof_thread_name_set(*(const char **)newp)) { + + tsd = tsd_tryget(); + if (tsd == NULL) { + ret = EAGAIN; + goto label_return; + } + + if (prof_thread_name_set(tsd, *(const char **)newp)) { ret = EAGAIN; goto label_return; } @@ -1675,6 +1719,7 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, { int ret; size_t lg_sample = lg_prof_sample; + tsd_t *tsd; if (config_prof == false) return (ENOENT); @@ -1684,7 +1729,13 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, if (lg_sample >= (sizeof(uint64_t) << 3)) lg_sample = (sizeof(uint64_t) << 3) - 1; - prof_reset(lg_sample); + tsd = tsd_tryget(); + if (tsd == NULL) { + ret = EAGAIN; + goto label_return; + } + + prof_reset(tsd, lg_sample); ret = 0; label_return: diff --git a/src/huge.c b/src/huge.c index 0b7db7fc..2e30ccfd 100644 --- a/src/huge.c +++ b/src/huge.c @@ -13,14 +13,15 @@ static malloc_mutex_t huge_mtx; static extent_tree_t huge; void * -huge_malloc(arena_t *arena, size_t size, bool zero) +huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero) { - return (huge_palloc(arena, size, chunksize, zero)); + return (huge_palloc(tsd, arena, size, chunksize, zero)); } void * -huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) +huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, + bool zero) { void *ret; size_t csize; @@ -45,7 +46,7 @@ huge_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) * it is possible to make correct junk/zero fill decisions below. */ is_zeroed = zero; - arena = choose_arena(arena); + arena = choose_arena(tsd, arena); ret = arena_chunk_alloc_huge(arena, csize, alignment, &is_zeroed); if (ret == NULL) { base_node_dalloc(node); @@ -90,7 +91,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) } void * -huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, +huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc) { void *ret; @@ -106,18 +107,18 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, * space and copying. */ if (alignment > chunksize) - ret = huge_palloc(arena, size + extra, alignment, zero); + ret = huge_palloc(tsd, arena, size + extra, alignment, zero); else - ret = huge_malloc(arena, size + extra, zero); + ret = huge_malloc(tsd, arena, size + extra, zero); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment > chunksize) - ret = huge_palloc(arena, size, alignment, zero); + ret = huge_palloc(tsd, arena, size, alignment, zero); else - ret = huge_malloc(arena, size, zero); + ret = huge_malloc(tsd, arena, size, zero); if (ret == NULL) return (NULL); @@ -129,7 +130,7 @@ huge_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); - iqalloc(ptr, try_tcache_dalloc); + iqalloc(tsd, ptr, try_tcache_dalloc); return (ret); } diff --git a/src/jemalloc.c b/src/jemalloc.c index c5b8f520..4d3b22e5 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -5,8 +5,6 @@ /* Data. */ malloc_tsd_data(, arenas, arena_t *, NULL) -malloc_tsd_data(, thread_allocated, thread_allocated_t, - THREAD_ALLOCATED_INITIALIZER) /* Runtime configuration options. */ const char *je_malloc_conf; @@ -142,7 +140,7 @@ arenas_extend(unsigned ind) /* Slow path, called only by choose_arena(). */ arena_t * -choose_arena_hard(void) +choose_arena_hard(tsd_t *tsd) { arena_t *ret; @@ -196,11 +194,32 @@ choose_arena_hard(void) malloc_mutex_unlock(&arenas_lock); } - arenas_tsd_set(&ret); + tsd_arena_set(tsd, ret); return (ret); } +void +thread_allocated_cleanup(tsd_t *tsd) +{ + + /* Do nothing. */ +} + +void +thread_deallocated_cleanup(tsd_t *tsd) +{ + + /* Do nothing. */ +} + +void +arena_cleanup(tsd_t *tsd) +{ + + /* Do nothing. */ +} + static void stats_print_atexit(void) { @@ -691,7 +710,11 @@ malloc_init_hard(void) #endif malloc_initializer = INITIALIZER; - malloc_tsd_boot(); + if (malloc_tsd_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (config_prof) prof_boot0(); @@ -726,7 +749,7 @@ malloc_init_hard(void) arena_boot(); - if (config_tcache && tcache_boot0()) { + if (config_tcache && tcache_boot()) { malloc_mutex_unlock(&init_lock); return (true); } @@ -759,27 +782,6 @@ malloc_init_hard(void) return (true); } - /* Initialize allocation counters before any allocations can occur. */ - if (config_stats && thread_allocated_tsd_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - - if (arenas_tsd_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - - if (config_tcache && tcache_boot1()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - - if (config_fill && quarantine_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - if (config_prof && prof_boot2()) { malloc_mutex_unlock(&init_lock); return (true); @@ -863,36 +865,36 @@ malloc_init_hard(void) */ static void * -imalloc_prof_sample(size_t usize, prof_tctx_t *tctx) +imalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) { void *p; if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = imalloc(LARGE_MINCLASS); + p = imalloc(tsd, LARGE_MINCLASS); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else - p = imalloc(usize); + p = imalloc(tsd, usize); return (p); } JEMALLOC_ALWAYS_INLINE_C void * -imalloc_prof(size_t usize) +imalloc_prof(tsd_t *tsd, size_t usize) { void *p; prof_tctx_t *tctx; - tctx = prof_alloc_prep(usize, true); + tctx = prof_alloc_prep(tsd, usize, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) - p = imalloc_prof_sample(usize, tctx); + p = imalloc_prof_sample(tsd, usize, tctx); else - p = imalloc(usize); + p = imalloc(tsd, usize); if (p == NULL) { - prof_alloc_rollback(tctx, true); + prof_alloc_rollback(tsd, tctx, true); return (NULL); } prof_malloc(p, usize, tctx); @@ -901,32 +903,33 @@ imalloc_prof(size_t usize) } JEMALLOC_ALWAYS_INLINE_C void * -imalloc_body(size_t size, size_t *usize) +imalloc_body(size_t size, tsd_t **tsd, size_t *usize) { - if (unlikely(malloc_init())) + if (unlikely(malloc_init()) || unlikely((*tsd = tsd_tryget()) == NULL)) return (NULL); if (config_prof && opt_prof) { *usize = s2u(size); - return (imalloc_prof(*usize)); + return (imalloc_prof(*tsd, *usize)); } if (config_stats || (config_valgrind && unlikely(in_valgrind))) *usize = s2u(size); - return (imalloc(size)); + return (imalloc(*tsd, size)); } void * je_malloc(size_t size) { void *ret; + tsd_t *tsd; size_t usize JEMALLOC_CC_SILENCE_INIT(0); if (size == 0) size = 1; - ret = imalloc_body(size, &usize); + ret = imalloc_body(size, &tsd, &usize); if (unlikely(ret == NULL)) { if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(": Error in malloc(): " @@ -937,7 +940,7 @@ je_malloc(size_t size) } if (config_stats && likely(ret != NULL)) { assert(usize == isalloc(ret, config_prof)); - thread_allocated_tsd_get()->allocated += usize; + *tsd_thread_allocatedp_get(tsd) += usize; } UTRACE(0, size, ret); JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); @@ -945,7 +948,8 @@ je_malloc(size_t size) } static void * -imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx) +imemalign_prof_sample(tsd_t *tsd, size_t alignment, size_t usize, + prof_tctx_t *tctx) { void *p; @@ -953,29 +957,29 @@ imemalign_prof_sample(size_t alignment, size_t usize, prof_tctx_t *tctx) return (NULL); if (usize <= SMALL_MAXCLASS) { assert(sa2u(LARGE_MINCLASS, alignment) == LARGE_MINCLASS); - p = imalloc(LARGE_MINCLASS); + p = imalloc(tsd, LARGE_MINCLASS); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else - p = ipalloc(usize, alignment, false); + p = ipalloc(tsd, usize, alignment, false); return (p); } JEMALLOC_ALWAYS_INLINE_C void * -imemalign_prof(size_t alignment, size_t usize) +imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) { void *p; prof_tctx_t *tctx; - tctx = prof_alloc_prep(usize, true); + tctx = prof_alloc_prep(tsd, usize, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) - p = imemalign_prof_sample(alignment, usize, tctx); + p = imemalign_prof_sample(tsd, alignment, usize, tctx); else - p = ipalloc(usize, alignment, false); + p = ipalloc(tsd, usize, alignment, false); if (p == NULL) { - prof_alloc_rollback(tctx, true); + prof_alloc_rollback(tsd, tctx, true); return (NULL); } prof_malloc(p, usize, tctx); @@ -988,12 +992,13 @@ static int imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) { int ret; + tsd_t *tsd; size_t usize; void *result; assert(min_alignment != 0); - if (unlikely(malloc_init())) { + if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) { result = NULL; goto label_oom; } else { @@ -1020,9 +1025,9 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) } if (config_prof && opt_prof) - result = imemalign_prof(alignment, usize); + result = imemalign_prof(tsd, alignment, usize); else - result = ipalloc(usize, alignment, false); + result = ipalloc(tsd, usize, alignment, false); if (unlikely(result == NULL)) goto label_oom; } @@ -1032,7 +1037,7 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) label_return: if (config_stats && likely(result != NULL)) { assert(usize == isalloc(result, config_prof)); - thread_allocated_tsd_get()->allocated += usize; + *tsd_thread_allocatedp_get(tsd) += usize; } UTRACE(0, size, result); return (ret); @@ -1072,36 +1077,36 @@ je_aligned_alloc(size_t alignment, size_t size) } static void * -icalloc_prof_sample(size_t usize, prof_tctx_t *tctx) +icalloc_prof_sample(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) { void *p; if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = icalloc(LARGE_MINCLASS); + p = icalloc(tsd, LARGE_MINCLASS); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else - p = icalloc(usize); + p = icalloc(tsd, usize); return (p); } JEMALLOC_ALWAYS_INLINE_C void * -icalloc_prof(size_t usize) +icalloc_prof(tsd_t *tsd, size_t usize) { void *p; prof_tctx_t *tctx; - tctx = prof_alloc_prep(usize, true); + tctx = prof_alloc_prep(tsd, usize, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) - p = icalloc_prof_sample(usize, tctx); + p = icalloc_prof_sample(tsd, usize, tctx); else - p = icalloc(usize); + p = icalloc(tsd, usize); if (p == NULL) { - prof_alloc_rollback(tctx, true); + prof_alloc_rollback(tsd, tctx, true); return (NULL); } prof_malloc(p, usize, tctx); @@ -1113,10 +1118,11 @@ void * je_calloc(size_t num, size_t size) { void *ret; + tsd_t *tsd; size_t num_size; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - if (unlikely(malloc_init())) { + if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) { num_size = 0; ret = NULL; goto label_return; @@ -1144,11 +1150,11 @@ je_calloc(size_t num, size_t size) if (config_prof && opt_prof) { usize = s2u(num_size); - ret = icalloc_prof(usize); + ret = icalloc_prof(tsd, usize); } else { if (config_stats || (config_valgrind && unlikely(in_valgrind))) usize = s2u(num_size); - ret = icalloc(num_size); + ret = icalloc(tsd, num_size); } label_return: @@ -1162,7 +1168,7 @@ label_return: } if (config_stats && likely(ret != NULL)) { assert(usize == isalloc(ret, config_prof)); - thread_allocated_tsd_get()->allocated += usize; + *tsd_thread_allocatedp_get(tsd) += usize; } UTRACE(0, num_size, ret); JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); @@ -1170,44 +1176,44 @@ label_return: } static void * -irealloc_prof_sample(void *oldptr, size_t usize, prof_tctx_t *tctx) +irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t usize, prof_tctx_t *tctx) { void *p; if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = iralloc(oldptr, LARGE_MINCLASS, 0, false); + p = iralloc(tsd, oldptr, LARGE_MINCLASS, 0, false); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else - p = iralloc(oldptr, usize, 0, false); + p = iralloc(tsd, oldptr, usize, 0, false); return (p); } JEMALLOC_ALWAYS_INLINE_C void * -irealloc_prof(void *oldptr, size_t old_usize, size_t usize) +irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) { void *p; prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(oldptr); - tctx = prof_alloc_prep(usize, true); + tctx = prof_alloc_prep(tsd, usize, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) - p = irealloc_prof_sample(oldptr, usize, tctx); + p = irealloc_prof_sample(tsd, oldptr, usize, tctx); else - p = iralloc(oldptr, usize, 0, false); + p = iralloc(tsd, oldptr, usize, 0, false); if (p == NULL) return (NULL); - prof_realloc(p, usize, tctx, true, old_usize, old_tctx); + prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx); return (p); } JEMALLOC_INLINE_C void -ifree(void *ptr, bool try_tcache) +ifree(tsd_t *tsd, void *ptr, bool try_tcache) { size_t usize; UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); @@ -1217,19 +1223,19 @@ ifree(void *ptr, bool try_tcache) if (config_prof && opt_prof) { usize = isalloc(ptr, config_prof); - prof_free(ptr, usize); + prof_free(tsd, ptr, usize); } else if (config_stats || config_valgrind) usize = isalloc(ptr, config_prof); - if (config_stats) - thread_allocated_tsd_get()->deallocated += usize; + if (config_stats && likely(tsd != NULL)) + *tsd_thread_deallocatedp_get(tsd) += usize; if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); - iqalloc(ptr, try_tcache); + iqalloc(tsd, ptr, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); } JEMALLOC_INLINE_C void -isfree(void *ptr, size_t usize, bool try_tcache) +isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache) { UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); @@ -1237,12 +1243,12 @@ isfree(void *ptr, size_t usize, bool try_tcache) assert(malloc_initialized || IS_INITIALIZER); if (config_prof && opt_prof) - prof_free(ptr, usize); - if (config_stats) - thread_allocated_tsd_get()->deallocated += usize; + prof_free(tsd, ptr, usize); + if (config_stats && likely(tsd != NULL)) + *tsd_thread_deallocatedp_get(tsd) += usize; if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); - isqalloc(ptr, usize, try_tcache); + isqalloc(tsd, ptr, usize, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); } @@ -1250,6 +1256,7 @@ void * je_realloc(void *ptr, size_t size) { void *ret; + tsd_t *tsd; size_t usize JEMALLOC_CC_SILENCE_INIT(0); size_t old_usize = 0; UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); @@ -1258,7 +1265,8 @@ je_realloc(void *ptr, size_t size) if (ptr != NULL) { /* realloc(ptr, 0) is equivalent to free(ptr). */ UTRACE(ptr, 0, 0); - ifree(ptr, true); + tsd = tsd_tryget(); + ifree(tsd, ptr, true); return (NULL); } size = 1; @@ -1268,24 +1276,29 @@ je_realloc(void *ptr, size_t size) assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); - if ((config_prof && opt_prof) || config_stats || - (config_valgrind && unlikely(in_valgrind))) - old_usize = isalloc(ptr, config_prof); - if (config_valgrind && unlikely(in_valgrind)) - old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); + if ((tsd = tsd_tryget()) != NULL) { + if ((config_prof && opt_prof) || config_stats || + (config_valgrind && unlikely(in_valgrind))) + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && unlikely(in_valgrind)) { + old_rzsize = config_prof ? p2rz(ptr) : + u2rz(old_usize); + } - if (config_prof && opt_prof) { - usize = s2u(size); - ret = irealloc_prof(ptr, old_usize, usize); - } else { - if (config_stats || (config_valgrind && - unlikely(in_valgrind))) + if (config_prof && opt_prof) { usize = s2u(size); - ret = iralloc(ptr, size, 0, false); - } + ret = irealloc_prof(tsd, ptr, old_usize, usize); + } else { + if (config_stats || (config_valgrind && + unlikely(in_valgrind))) + usize = s2u(size); + ret = iralloc(tsd, ptr, size, 0, false); + } + } else + ret = NULL; } else { /* realloc(NULL, size) is equivalent to malloc(size). */ - ret = imalloc_body(size, &usize); + ret = imalloc_body(size, &tsd, &usize); } if (unlikely(ret == NULL)) { @@ -1297,11 +1310,11 @@ je_realloc(void *ptr, size_t size) set_errno(ENOMEM); } if (config_stats && likely(ret != NULL)) { - thread_allocated_t *ta; assert(usize == isalloc(ret, config_prof)); - ta = thread_allocated_tsd_get(); - ta->allocated += usize; - ta->deallocated += old_usize; + if (tsd != NULL) { + *tsd_thread_allocatedp_get(tsd) += usize; + *tsd_thread_deallocatedp_get(tsd) += old_usize; + } } UTRACE(ptr, size, ret); JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, @@ -1315,7 +1328,7 @@ je_free(void *ptr) UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) - ifree(ptr, true); + ifree(tsd_tryget(), ptr, true); } /* @@ -1425,50 +1438,52 @@ imallocx_flags_decode(size_t size, int flags, size_t *usize, size_t *alignment, } JEMALLOC_ALWAYS_INLINE_C void * -imallocx_flags(size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena) +imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero, + bool try_tcache, arena_t *arena) { - if (alignment != 0) - return (ipalloct(usize, alignment, zero, try_tcache, arena)); + if (alignment != 0) { + return (ipalloct(tsd, usize, alignment, zero, try_tcache, + arena)); + } if (zero) - return (icalloct(usize, try_tcache, arena)); - return (imalloct(usize, try_tcache, arena)); + return (icalloct(tsd, usize, try_tcache, arena)); + return (imalloct(tsd, usize, try_tcache, arena)); } JEMALLOC_ALWAYS_INLINE_C void * -imallocx_maybe_flags(size_t size, int flags, size_t usize, size_t alignment, - bool zero, bool try_tcache, arena_t *arena) +imallocx_maybe_flags(tsd_t *tsd, size_t size, int flags, size_t usize, + size_t alignment, bool zero, bool try_tcache, arena_t *arena) { if (likely(flags == 0)) - return (imalloc(size)); - return (imallocx_flags(usize, alignment, zero, try_tcache, arena)); + return (imalloc(tsd, size)); + return (imallocx_flags(tsd, usize, alignment, zero, try_tcache, arena)); } static void * -imallocx_prof_sample(size_t size, int flags, size_t usize, size_t alignment, - bool zero, bool try_tcache, arena_t *arena) +imallocx_prof_sample(tsd_t *tsd, size_t size, int flags, size_t usize, + size_t alignment, bool zero, bool try_tcache, arena_t *arena) { void *p; if (usize <= SMALL_MAXCLASS) { assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); - p = imalloct(LARGE_MINCLASS, try_tcache, arena); + p = imalloct(tsd, LARGE_MINCLASS, try_tcache, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else { - p = imallocx_maybe_flags(size, flags, usize, alignment, zero, - try_tcache, arena); + p = imallocx_maybe_flags(tsd, size, flags, usize, alignment, + zero, try_tcache, arena); } return (p); } JEMALLOC_ALWAYS_INLINE_C void * -imallocx_prof(size_t size, int flags, size_t *usize) +imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) { void *p; size_t alignment; @@ -1479,17 +1494,17 @@ imallocx_prof(size_t size, int flags, size_t *usize) imallocx_flags_decode(size, flags, usize, &alignment, &zero, &try_tcache, &arena); - tctx = prof_alloc_prep(*usize, true); + tctx = prof_alloc_prep(tsd, *usize, true); if (likely((uintptr_t)tctx == (uintptr_t)1U)) { - p = imallocx_maybe_flags(size, flags, *usize, alignment, zero, - try_tcache, arena); + p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment, + zero, try_tcache, arena); } else if ((uintptr_t)tctx > (uintptr_t)1U) { - p = imallocx_prof_sample(size, flags, *usize, alignment, zero, - try_tcache, arena); + p = imallocx_prof_sample(tsd, size, flags, *usize, alignment, + zero, try_tcache, arena); } else p = NULL; if (unlikely(p == NULL)) { - prof_alloc_rollback(tctx, true); + prof_alloc_rollback(tsd, tctx, true); return (NULL); } prof_malloc(p, *usize, tctx); @@ -1498,7 +1513,7 @@ imallocx_prof(size_t size, int flags, size_t *usize) } JEMALLOC_ALWAYS_INLINE_C void * -imallocx_no_prof(size_t size, int flags, size_t *usize) +imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) { size_t alignment; bool zero; @@ -1508,35 +1523,39 @@ imallocx_no_prof(size_t size, int flags, size_t *usize) if (likely(flags == 0)) { if (config_stats || (config_valgrind && unlikely(in_valgrind))) *usize = s2u(size); - return (imalloc(size)); + return (imalloc(tsd, size)); } imallocx_flags_decode_hard(size, flags, usize, &alignment, &zero, &try_tcache, &arena); - return (imallocx_flags(*usize, alignment, zero, try_tcache, arena)); + return (imallocx_flags(tsd, *usize, alignment, zero, try_tcache, + arena)); } void * je_mallocx(size_t size, int flags) { + tsd_t *tsd; void *p; size_t usize; assert(size != 0); - if (unlikely(malloc_init())) + if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) goto label_oom; if (config_prof && opt_prof) - p = imallocx_prof(size, flags, &usize); + p = imallocx_prof(tsd, size, flags, &usize); else - p = imallocx_no_prof(size, flags, &usize); + p = imallocx_no_prof(tsd, size, flags, &usize); if (unlikely(p == NULL)) goto label_oom; if (config_stats) { + tsd_t *tsd = tsd_tryget(); assert(usize == isalloc(p, config_prof)); - thread_allocated_tsd_get()->allocated += usize; + if (tsd != NULL) + *tsd_thread_allocatedp_get(tsd) += usize; } UTRACE(0, size, p); JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags)); @@ -1551,47 +1570,47 @@ label_oom: } static void * -irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, - bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena, - prof_tctx_t *tctx) +irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t size, size_t alignment, + size_t usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena, prof_tctx_t *tctx) { void *p; if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = iralloct(oldptr, LARGE_MINCLASS, alignment, zero, + p = iralloct(tsd, oldptr, LARGE_MINCLASS, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else { - p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc, - try_tcache_dalloc, arena); + p = iralloct(tsd, oldptr, size, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); } return (p); } JEMALLOC_ALWAYS_INLINE_C void * -irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, - size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, - arena_t *arena) +irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, + size_t alignment, size_t *usize, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc, arena_t *arena) { void *p; prof_tctx_t *old_tctx, *tctx; old_tctx = prof_tctx_get(oldptr); - tctx = prof_alloc_prep(*usize, false); + tctx = prof_alloc_prep(tsd, *usize, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, - try_tcache_alloc, try_tcache_dalloc, arena, tctx); + p = irallocx_prof_sample(tsd, oldptr, size, alignment, *usize, + zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx); } else { - p = iralloct(oldptr, size, alignment, zero, try_tcache_alloc, - try_tcache_dalloc, arena); + p = iralloct(tsd, oldptr, size, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); } if (unlikely(p == NULL)) { - prof_alloc_rollback(tctx, false); + prof_alloc_rollback(tsd, tctx, false); return (NULL); } @@ -1606,7 +1625,7 @@ irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, */ *usize = isalloc(p, config_prof); } - prof_realloc(p, *usize, tctx, false, old_usize, old_tctx); + prof_realloc(tsd, p, *usize, tctx, false, old_usize, old_tctx); return (p); } @@ -1615,6 +1634,7 @@ void * je_rallocx(void *ptr, size_t size, int flags) { void *p; + tsd_t *tsd; size_t usize; UNUSED size_t old_usize JEMALLOC_CC_SILENCE_INIT(0); UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); @@ -1628,6 +1648,9 @@ je_rallocx(void *ptr, size_t size, int flags) assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); + if (unlikely((tsd = tsd_tryget()) == NULL)) + goto label_oom; + if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk; @@ -1651,12 +1674,12 @@ je_rallocx(void *ptr, size_t size, int flags) if (config_prof && opt_prof) { usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); assert(usize != 0); - p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, - try_tcache_alloc, try_tcache_dalloc, arena); + p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, + zero, try_tcache_alloc, try_tcache_dalloc, arena); if (unlikely(p == NULL)) goto label_oom; } else { - p = iralloct(ptr, size, alignment, zero, try_tcache_alloc, + p = iralloct(tsd, ptr, size, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); if (unlikely(p == NULL)) goto label_oom; @@ -1665,10 +1688,8 @@ je_rallocx(void *ptr, size_t size, int flags) } if (config_stats) { - thread_allocated_t *ta; - ta = thread_allocated_tsd_get(); - ta->allocated += usize; - ta->deallocated += old_usize; + *tsd_thread_allocatedp_get(tsd) += usize; + *tsd_thread_deallocatedp_get(tsd) += old_usize; } UTRACE(ptr, size, p); JEMALLOC_VALGRIND_REALLOC(true, p, usize, false, ptr, old_usize, @@ -1724,8 +1745,8 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, } JEMALLOC_ALWAYS_INLINE_C size_t -ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, - size_t alignment, bool zero, arena_t *arena) +ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, + size_t extra, size_t alignment, bool zero, arena_t *arena) { size_t max_usize, usize; prof_tctx_t *old_tctx, *tctx; @@ -1739,7 +1760,7 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, */ max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment); - tctx = prof_alloc_prep(max_usize, false); + tctx = prof_alloc_prep(tsd, max_usize, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, alignment, zero, max_usize, arena, tctx); @@ -1748,10 +1769,10 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, zero, arena); } if (unlikely(usize == old_usize)) { - prof_alloc_rollback(tctx, false); + prof_alloc_rollback(tsd, tctx, false); return (usize); } - prof_realloc(ptr, usize, tctx, false, old_usize, old_tctx); + prof_realloc(tsd, ptr, usize, tctx, false, old_usize, old_tctx); return (usize); } @@ -1759,6 +1780,7 @@ ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, size_t je_xallocx(void *ptr, size_t size, size_t extra, int flags) { + tsd_t *tsd; size_t usize, old_usize; UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t alignment = MALLOCX_ALIGN_GET(flags); @@ -1778,12 +1800,16 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) arena = NULL; old_usize = isalloc(ptr, config_prof); + if (unlikely((tsd = tsd_tryget()) == NULL)) { + usize = old_usize; + goto label_not_resized; + } if (config_valgrind && unlikely(in_valgrind)) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { - usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, - zero, arena); + usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, + alignment, zero, arena); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, zero, arena); @@ -1792,10 +1818,8 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) goto label_not_resized; if (config_stats) { - thread_allocated_t *ta; - ta = thread_allocated_tsd_get(); - ta->allocated += usize; - ta->deallocated += old_usize; + *tsd_thread_allocatedp_get(tsd) += usize; + *tsd_thread_deallocatedp_get(tsd) += old_usize; } JEMALLOC_VALGRIND_REALLOC(false, ptr, usize, false, ptr, old_usize, old_rzsize, false, zero); @@ -1839,7 +1863,7 @@ je_dallocx(void *ptr, int flags) try_tcache = true; UTRACE(ptr, 0, 0); - ifree(ptr, try_tcache); + ifree(tsd_tryget(), ptr, try_tcache); } JEMALLOC_ALWAYS_INLINE_C size_t @@ -1875,7 +1899,7 @@ je_sdallocx(void *ptr, size_t size, int flags) try_tcache = true; UTRACE(ptr, 0, 0); - isfree(ptr, usize, try_tcache); + isfree(tsd_tryget(), ptr, usize, try_tcache); } size_t @@ -2072,9 +2096,9 @@ a0alloc(size_t size, bool zero) size = 1; if (size <= arena_maxclass) - return (arena_malloc(arenas[0], size, zero, false)); + return (arena_malloc(NULL, arenas[0], size, zero, false)); else - return (huge_malloc(NULL, size, zero)); + return (huge_malloc(NULL, arenas[0], size, zero)); } void * @@ -2101,7 +2125,7 @@ a0free(void *ptr) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(chunk, ptr, false); + arena_dalloc(NULL, chunk, ptr, false); else huge_dalloc(ptr); } diff --git a/src/prof.c b/src/prof.c index a773e224..dd84f533 100644 --- a/src/prof.c +++ b/src/prof.c @@ -14,8 +14,6 @@ /******************************************************************************/ /* Data. */ -malloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL) - bool opt_prof = false; bool opt_prof_active = true; size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; @@ -102,9 +100,9 @@ static bool prof_booted = false; */ static bool prof_tctx_should_destroy(prof_tctx_t *tctx); -static void prof_tctx_destroy(prof_tctx_t *tctx); +static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); static bool prof_tdata_should_destroy(prof_tdata_t *tdata); -static void prof_tdata_destroy(prof_tdata_t *tdata); +static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata); /******************************************************************************/ /* Red-black trees. */ @@ -151,7 +149,7 @@ rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, /******************************************************************************/ void -prof_alloc_rollback(prof_tctx_t *tctx, bool updated) +prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) { prof_tdata_t *tdata; @@ -164,8 +162,8 @@ prof_alloc_rollback(prof_tctx_t *tctx, bool updated) * potential for sample bias is minimal except in contrived * programs. */ - tdata = prof_tdata_get(true); - if ((uintptr_t)tdata > (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(tsd, true); + if (tdata != NULL) prof_sample_threshold_update(tctx->tdata); } @@ -173,7 +171,7 @@ prof_alloc_rollback(prof_tctx_t *tctx, bool updated) malloc_mutex_lock(tctx->tdata->lock); tctx->prepared = false; if (prof_tctx_should_destroy(tctx)) - prof_tctx_destroy(tctx); + prof_tctx_destroy(tsd, tctx); else malloc_mutex_unlock(tctx->tdata->lock); } @@ -195,7 +193,7 @@ prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) { } void -prof_free_sampled_object(size_t usize, prof_tctx_t *tctx) +prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) { malloc_mutex_lock(tctx->tdata->lock); @@ -205,7 +203,7 @@ prof_free_sampled_object(size_t usize, prof_tctx_t *tctx) tctx->cnts.curbytes -= usize; if (prof_tctx_should_destroy(tctx)) - prof_tctx_destroy(tctx); + prof_tctx_destroy(tsd, tctx); else malloc_mutex_unlock(tctx->tdata->lock); } @@ -494,13 +492,13 @@ prof_tdata_mutex_choose(uint64_t thr_uid) } static prof_gctx_t * -prof_gctx_create(prof_bt_t *bt) +prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) { /* * Create a single allocation that has space for vec of length bt->len. */ - prof_gctx_t *gctx = (prof_gctx_t *)imalloc(offsetof(prof_gctx_t, vec) + - (bt->len * sizeof(void *))); + prof_gctx_t *gctx = (prof_gctx_t *)imalloc(tsd, offsetof(prof_gctx_t, + vec) + (bt->len * sizeof(void *))); if (gctx == NULL) return (NULL); gctx->lock = prof_gctx_mutex_choose(); @@ -518,7 +516,7 @@ prof_gctx_create(prof_bt_t *bt) } static void -prof_gctx_maybe_destroy(prof_gctx_t *gctx, prof_tdata_t *tdata) +prof_gctx_maybe_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata) { cassert(config_prof); @@ -534,12 +532,12 @@ prof_gctx_maybe_destroy(prof_gctx_t *gctx, prof_tdata_t *tdata) malloc_mutex_lock(gctx->lock); if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { /* Remove gctx from bt2gctx. */ - if (ckh_remove(&bt2gctx, &gctx->bt, NULL, NULL)) + if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) not_reached(); prof_leave(tdata); /* Destroy gctx. */ malloc_mutex_unlock(gctx->lock); - idalloc(gctx); + idalloc(tsd, gctx); } else { /* * Compensate for increment in prof_tctx_destroy() or @@ -580,7 +578,7 @@ prof_gctx_should_destroy(prof_gctx_t *gctx) /* tctx->tdata->lock is held upon entry, and released before return. */ static void -prof_tctx_destroy(prof_tctx_t *tctx) +prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) { prof_tdata_t *tdata = tctx->tdata; prof_gctx_t *gctx = tctx->gctx; @@ -592,7 +590,7 @@ prof_tctx_destroy(prof_tctx_t *tctx) assert(tctx->cnts.accumobjs == 0); assert(tctx->cnts.accumbytes == 0); - ckh_remove(&tdata->bt2tctx, &gctx->bt, NULL, NULL); + ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); destroy_tdata = prof_tdata_should_destroy(tdata); malloc_mutex_unlock(tdata->lock); @@ -618,17 +616,17 @@ prof_tctx_destroy(prof_tctx_t *tctx) destroy_gctx = false; malloc_mutex_unlock(gctx->lock); if (destroy_gctx) - prof_gctx_maybe_destroy(gctx, tdata); + prof_gctx_maybe_destroy(tsd, gctx, tdata); if (destroy_tdata) - prof_tdata_destroy(tdata); + prof_tdata_destroy(tsd, tdata); - idalloc(tctx); + idalloc(tsd, tctx); } static bool -prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey, - prof_gctx_t **p_gctx, bool *p_new_gctx) +prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, + void **p_btkey, prof_gctx_t **p_gctx, bool *p_new_gctx) { union { prof_gctx_t *p; @@ -643,16 +641,16 @@ prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey, prof_enter(tdata); if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { /* bt has never been seen before. Insert it. */ - gctx.p = prof_gctx_create(bt); + gctx.p = prof_gctx_create(tsd, bt); if (gctx.v == NULL) { prof_leave(tdata); return (true); } btkey.p = &gctx.p->bt; - if (ckh_insert(&bt2gctx, btkey.v, gctx.v)) { + if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { /* OOM. */ prof_leave(tdata); - idalloc(gctx.v); + idalloc(tsd, gctx.v); return (true); } new_gctx = true; @@ -675,7 +673,7 @@ prof_lookup_global(prof_bt_t *bt, prof_tdata_t *tdata, void **p_btkey, } prof_tctx_t * -prof_lookup(prof_bt_t *bt) +prof_lookup(tsd_t *tsd, prof_bt_t *bt) { union { prof_tctx_t *p; @@ -686,8 +684,8 @@ prof_lookup(prof_bt_t *bt) cassert(config_prof); - tdata = prof_tdata_get(false); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(tsd, false); + if (tdata == NULL) return (NULL); malloc_mutex_lock(tdata->lock); @@ -704,15 +702,15 @@ prof_lookup(prof_bt_t *bt) * This thread's cache lacks bt. Look for it in the global * cache. */ - if (prof_lookup_global(bt, tdata, &btkey, &gctx, + if (prof_lookup_global(tsd, bt, tdata, &btkey, &gctx, &new_gctx)) return (NULL); /* Link a prof_tctx_t into gctx for this thread. */ - ret.v = imalloc(sizeof(prof_tctx_t)); + ret.v = imalloc(tsd, sizeof(prof_tctx_t)); if (ret.p == NULL) { if (new_gctx) - prof_gctx_maybe_destroy(gctx, tdata); + prof_gctx_maybe_destroy(tsd, gctx, tdata); return (NULL); } ret.p->tdata = tdata; @@ -721,12 +719,12 @@ prof_lookup(prof_bt_t *bt) ret.p->prepared = true; ret.p->state = prof_tctx_state_nominal; malloc_mutex_lock(tdata->lock); - error = ckh_insert(&tdata->bt2tctx, btkey, ret.v); + error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v); malloc_mutex_unlock(tdata->lock); if (error) { if (new_gctx) - prof_gctx_maybe_destroy(gctx, tdata); - idalloc(ret.v); + prof_gctx_maybe_destroy(tsd, gctx, tdata); + idalloc(tsd, ret.v); return (NULL); } malloc_mutex_lock(gctx->lock); @@ -798,10 +796,13 @@ size_t prof_bt_count(void) { size_t bt_count; + tsd_t *tsd; prof_tdata_t *tdata; - tdata = prof_tdata_get(false); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + if ((tsd = tsd_tryget()) == NULL) + return (0); + tdata = prof_tdata_get(tsd, false); + if (tdata == NULL) return (0); prof_enter(tdata); @@ -989,6 +990,7 @@ static prof_tctx_t * prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) { prof_tctx_t *ret; + tsd_t *tsd = (tsd_t *)arg; switch (tctx->state) { case prof_tctx_state_nominal: @@ -1000,7 +1002,7 @@ prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) case prof_tctx_state_purgatory: ret = tctx_tree_next(tctxs, tctx); tctx_tree_remove(tctxs, tctx); - idalloc(tctx); + idalloc(tsd, tctx); goto label_return; default: not_reached(); @@ -1049,7 +1051,8 @@ prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) static prof_gctx_t * prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) { - prof_tdata_t *tdata = (prof_tdata_t *)arg; + tsd_t *tsd = (tsd_t *)arg; + prof_tdata_t *tdata = prof_tdata_get(tsd, false); prof_tctx_t *next; bool destroy_gctx; @@ -1057,13 +1060,13 @@ prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) next = NULL; do { next = tctx_tree_iter(&gctx->tctxs, next, prof_tctx_finish_iter, - NULL); + tsd); } while (next != NULL); gctx->nlimbo--; destroy_gctx = prof_gctx_should_destroy(gctx); malloc_mutex_unlock(gctx->lock); if (destroy_gctx) - prof_gctx_maybe_destroy(gctx, tdata); + prof_gctx_maybe_destroy(tsd, gctx, tdata); return (NULL); } @@ -1277,7 +1280,7 @@ label_return: } static bool -prof_dump(bool propagate_err, const char *filename, bool leakcheck) +prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) { prof_tdata_t *tdata; prof_cnt_t cnt_all; @@ -1291,8 +1294,8 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) cassert(config_prof); - tdata = prof_tdata_get(false); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(tsd, false); + if (tdata == NULL) return (true); malloc_mutex_lock(&prof_dump_mtx); @@ -1341,7 +1344,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) if (prof_dump_close(propagate_err)) goto label_open_close_error; - gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata); + gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd); malloc_mutex_unlock(&prof_dump_mtx); if (leakcheck) @@ -1351,7 +1354,7 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) label_write_error: prof_dump_close(propagate_err); label_open_close_error: - gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tdata); + gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd); malloc_mutex_unlock(&prof_dump_mtx); return (true); } @@ -1381,24 +1384,28 @@ prof_dump_filename(char *filename, char v, uint64_t vseq) static void prof_fdump(void) { + tsd_t *tsd; char filename[DUMP_FILENAME_BUFSIZE]; cassert(config_prof); if (prof_booted == false) return; + if ((tsd = tsd_tryget()) == NULL) + return; if (opt_prof_final && opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); prof_dump_filename(filename, 'f', VSEQ_INVALID); malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(false, filename, opt_prof_leak); + prof_dump(tsd, false, filename, opt_prof_leak); } } void prof_idump(void) { + tsd_t *tsd; prof_tdata_t *tdata; char filename[PATH_MAX + 1]; @@ -1406,8 +1413,10 @@ prof_idump(void) if (prof_booted == false) return; - tdata = prof_tdata_get(false); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + if ((tsd = tsd_tryget()) == NULL) + return; + tdata = prof_tdata_get(tsd, false); + if (tdata == NULL) return; if (tdata->enq) { tdata->enq_idump = true; @@ -1419,19 +1428,22 @@ prof_idump(void) prof_dump_filename(filename, 'i', prof_dump_iseq); prof_dump_iseq++; malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(false, filename, false); + prof_dump(tsd, false, filename, false); } } bool prof_mdump(const char *filename) { + tsd_t *tsd; char filename_buf[DUMP_FILENAME_BUFSIZE]; cassert(config_prof); if (opt_prof == false || prof_booted == false) return (true); + if ((tsd = tsd_tryget()) == NULL) + return (true); if (filename == NULL) { /* No filename specified, so automatically generate one. */ @@ -1443,12 +1455,13 @@ prof_mdump(const char *filename) malloc_mutex_unlock(&prof_dump_seq_mtx); filename = filename_buf; } - return (prof_dump(true, filename, false)); + return (prof_dump(tsd, true, filename, false)); } void prof_gdump(void) { + tsd_t *tsd; prof_tdata_t *tdata; char filename[DUMP_FILENAME_BUFSIZE]; @@ -1456,8 +1469,10 @@ prof_gdump(void) if (prof_booted == false) return; - tdata = prof_tdata_get(false); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + if ((tsd = tsd_tryget()) == NULL) + return; + tdata = prof_tdata_get(tsd, false); + if (tdata == NULL) return; if (tdata->enq) { tdata->enq_gdump = true; @@ -1469,7 +1484,7 @@ prof_gdump(void) prof_dump_filename(filename, 'u', prof_dump_useq); prof_dump_useq++; malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(false, filename, false); + prof_dump(tsd, false, filename, false); } } @@ -1510,14 +1525,14 @@ prof_thr_uid_alloc(void) } static prof_tdata_t * -prof_tdata_init_impl(uint64_t thr_uid) +prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid) { prof_tdata_t *tdata; cassert(config_prof); /* Initialize an empty cache for this thread. */ - tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); + tdata = (prof_tdata_t *)imalloc(tsd, sizeof(prof_tdata_t)); if (tdata == NULL) return (NULL); @@ -1526,9 +1541,9 @@ prof_tdata_init_impl(uint64_t thr_uid) tdata->thread_name = NULL; tdata->state = prof_tdata_state_attached; - if (ckh_new(&tdata->bt2tctx, PROF_CKH_MINITEMS, + if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) { - idalloc(tdata); + idalloc(tsd, tdata); return (NULL); } @@ -1542,8 +1557,6 @@ prof_tdata_init_impl(uint64_t thr_uid) tdata->dumping = false; tdata->active = true; - prof_tdata_tsd_set(&tdata); - malloc_mutex_lock(&tdatas_mtx); tdata_tree_insert(&tdatas, tdata); malloc_mutex_unlock(&tdatas_mtx); @@ -1552,17 +1565,17 @@ prof_tdata_init_impl(uint64_t thr_uid) } prof_tdata_t * -prof_tdata_init(void) +prof_tdata_init(tsd_t *tsd) { - return (prof_tdata_init_impl(prof_thr_uid_alloc())); + return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc())); } prof_tdata_t * -prof_tdata_reinit(prof_tdata_t *tdata) +prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) { - return (prof_tdata_init_impl(tdata->thr_uid)); + return (prof_tdata_init_impl(tsd, tdata->thr_uid)); } /* tdata->lock must be held. */ @@ -1578,7 +1591,7 @@ prof_tdata_should_destroy(prof_tdata_t *tdata) } static void -prof_tdata_destroy(prof_tdata_t *tdata) +prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata) { assert(prof_tdata_should_destroy(tdata)); @@ -1588,13 +1601,14 @@ prof_tdata_destroy(prof_tdata_t *tdata) malloc_mutex_unlock(&tdatas_mtx); if (tdata->thread_name != NULL) - idalloc(tdata->thread_name); - ckh_delete(&tdata->bt2tctx); - idalloc(tdata); + idalloc(tsd, tdata->thread_name); + ckh_delete(tsd, &tdata->bt2tctx); + idalloc(tsd, tdata); } static void -prof_tdata_state_transition(prof_tdata_t *tdata, prof_tdata_state_t state) +prof_tdata_state_transition(tsd_t *tsd, prof_tdata_t *tdata, + prof_tdata_state_t state) { bool destroy_tdata; @@ -1606,33 +1620,34 @@ prof_tdata_state_transition(prof_tdata_t *tdata, prof_tdata_state_t state) destroy_tdata = false; malloc_mutex_unlock(tdata->lock); if (destroy_tdata) - prof_tdata_destroy(tdata); + prof_tdata_destroy(tsd, tdata); } static void -prof_tdata_detach(prof_tdata_t *tdata) +prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) { - prof_tdata_state_transition(tdata, prof_tdata_state_detached); + prof_tdata_state_transition(tsd, tdata, prof_tdata_state_detached); } static void -prof_tdata_expire(prof_tdata_t *tdata) +prof_tdata_expire(tsd_t *tsd, prof_tdata_t *tdata) { - prof_tdata_state_transition(tdata, prof_tdata_state_expired); + prof_tdata_state_transition(tsd, tdata, prof_tdata_state_expired); } static prof_tdata_t * prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) { + tsd_t *tsd = (tsd_t *)arg; - prof_tdata_expire(tdata); + prof_tdata_expire(tsd, tdata); return (NULL); } void -prof_reset(size_t lg_sample) +prof_reset(tsd_t *tsd, size_t lg_sample) { assert(lg_sample < (sizeof(uint64_t) << 3)); @@ -1641,69 +1656,58 @@ prof_reset(size_t lg_sample) malloc_mutex_lock(&tdatas_mtx); lg_prof_sample = lg_sample; - tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, NULL); + tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, tsd); malloc_mutex_unlock(&tdatas_mtx); malloc_mutex_unlock(&prof_dump_mtx); } void -prof_tdata_cleanup(void *arg) +prof_tdata_cleanup(tsd_t *tsd) { - prof_tdata_t *tdata = *(prof_tdata_t **)arg; + prof_tdata_t *tdata; - cassert(config_prof); + if (!config_prof) + return; - if (tdata == PROF_TDATA_STATE_REINCARNATED) { - /* - * Another destructor deallocated memory after this destructor - * was called. Reset tdata to PROF_TDATA_STATE_PURGATORY in - * order to receive another callback. - */ - tdata = PROF_TDATA_STATE_PURGATORY; - prof_tdata_tsd_set(&tdata); - } else if (tdata == PROF_TDATA_STATE_PURGATORY) { - /* - * The previous time this destructor was called, we set the key - * to PROF_TDATA_STATE_PURGATORY so that other destructors - * wouldn't cause re-creation of the tdata. This time, do - * nothing, so that the destructor will not be called again. - */ - } else if (tdata != NULL) { - prof_tdata_detach(tdata); - tdata = PROF_TDATA_STATE_PURGATORY; - prof_tdata_tsd_set(&tdata); - } + tdata = tsd_prof_tdata_get(tsd); + if (tdata != NULL) + prof_tdata_detach(tsd, tdata); } const char * prof_thread_name_get(void) { - prof_tdata_t *tdata = prof_tdata_get(true); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tsd_t *tsd; + prof_tdata_t *tdata; + + if ((tsd = tsd_tryget()) == NULL) + return (NULL); + tdata = prof_tdata_get(tsd, true); + if (tdata == NULL) return (NULL); return (tdata->thread_name); } bool -prof_thread_name_set(const char *thread_name) +prof_thread_name_set(tsd_t *tsd, const char *thread_name) { prof_tdata_t *tdata; size_t size; char *s; - tdata = prof_tdata_get(true); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tdata = prof_tdata_get(tsd, true); + if (tdata == NULL) return (true); size = strlen(thread_name) + 1; - s = imalloc(size); + s = imalloc(tsd, size); if (s == NULL) return (true); memcpy(s, thread_name, size); if (tdata->thread_name != NULL) - idalloc(tdata->thread_name); + idalloc(tsd, tdata->thread_name); tdata->thread_name = s; return (false); } @@ -1711,8 +1715,13 @@ prof_thread_name_set(const char *thread_name) bool prof_thread_active_get(void) { - prof_tdata_t *tdata = prof_tdata_get(true); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + tsd_t *tsd; + prof_tdata_t *tdata; + + if ((tsd = tsd_tryget()) == NULL) + return (false); + tdata = prof_tdata_get(tsd, true); + if (tdata == NULL) return (false); return (tdata->active); } @@ -1720,10 +1729,13 @@ prof_thread_active_get(void) bool prof_thread_active_set(bool active) { + tsd_t *tsd; prof_tdata_t *tdata; - tdata = prof_tdata_get(true); - if ((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + if ((tsd = tsd_tryget()) == NULL) + return (true); + tdata = prof_tdata_get(tsd, true); + if (tdata == NULL) return (true); tdata->active = active; return (false); @@ -1772,20 +1784,18 @@ prof_boot2(void) cassert(config_prof); if (opt_prof) { + tsd_t *tsd; unsigned i; lg_prof_sample = opt_lg_prof_sample; - if (ckh_new(&bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, + if ((tsd = tsd_tryget()) == NULL) + return (true); + if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) return (true); if (malloc_mutex_init(&bt2gctx_mtx)) return (true); - if (prof_tdata_tsd_boot()) { - malloc_write( - ": Error in pthread_key_create()\n"); - abort(); - } tdata_tree_new(&tdatas); if (malloc_mutex_init(&tdatas_mtx)) diff --git a/src/quarantine.c b/src/quarantine.c index efddeae7..1301b479 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -9,26 +9,22 @@ #define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) #define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY -/******************************************************************************/ -/* Data. */ - -malloc_tsd_data(, quarantine, quarantine_t *, NULL) - /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static quarantine_t *quarantine_grow(quarantine_t *quarantine); -static void quarantine_drain_one(quarantine_t *quarantine); -static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); +static quarantine_t *quarantine_grow(tsd_t *tsd, quarantine_t *quarantine); +static void quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine); +static void quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, + size_t upper_bound); /******************************************************************************/ quarantine_t * -quarantine_init(size_t lg_maxobjs) +quarantine_init(tsd_t *tsd, size_t lg_maxobjs) { quarantine_t *quarantine; - quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + + quarantine = (quarantine_t *)imalloc(tsd, offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); if (quarantine == NULL) return (NULL); @@ -37,19 +33,17 @@ quarantine_init(size_t lg_maxobjs) quarantine->first = 0; quarantine->lg_maxobjs = lg_maxobjs; - quarantine_tsd_set(&quarantine); - return (quarantine); } static quarantine_t * -quarantine_grow(quarantine_t *quarantine) +quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) { quarantine_t *ret; - ret = quarantine_init(quarantine->lg_maxobjs + 1); + ret = quarantine_init(tsd, quarantine->lg_maxobjs + 1); if (ret == NULL) { - quarantine_drain_one(quarantine); + quarantine_drain_one(tsd, quarantine); return (quarantine); } @@ -71,17 +65,17 @@ quarantine_grow(quarantine_t *quarantine) memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * sizeof(quarantine_obj_t)); } - idalloc(quarantine); + idalloc(tsd, quarantine); return (ret); } static void -quarantine_drain_one(quarantine_t *quarantine) +quarantine_drain_one(tsd_t *tsd, quarantine_t *quarantine) { quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; assert(obj->usize == isalloc(obj->ptr, config_prof)); - idalloc(obj->ptr); + idalloc(tsd, obj->ptr); quarantine->curbytes -= obj->usize; quarantine->curobjs--; quarantine->first = (quarantine->first + 1) & ((ZU(1) << @@ -89,15 +83,15 @@ quarantine_drain_one(quarantine_t *quarantine) } static void -quarantine_drain(quarantine_t *quarantine, size_t upper_bound) +quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, size_t upper_bound) { while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) - quarantine_drain_one(quarantine); + quarantine_drain_one(tsd, quarantine); } void -quarantine(void *ptr) +quarantine(tsd_t *tsd, void *ptr) { quarantine_t *quarantine; size_t usize = isalloc(ptr, config_prof); @@ -105,17 +99,8 @@ quarantine(void *ptr) cassert(config_fill); assert(opt_quarantine); - quarantine = *quarantine_tsd_get(); - if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { - if (quarantine == QUARANTINE_STATE_PURGATORY) { - /* - * Make a note that quarantine() was called after - * quarantine_cleanup() was called. - */ - quarantine = QUARANTINE_STATE_REINCARNATED; - quarantine_tsd_set(&quarantine); - } - idalloc(ptr); + if ((quarantine = tsd_quarantine_get(tsd)) == NULL) { + idalloc(tsd, ptr); return; } /* @@ -125,11 +110,11 @@ quarantine(void *ptr) if (quarantine->curbytes + usize > opt_quarantine) { size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine - usize : 0; - quarantine_drain(quarantine, upper_bound); + quarantine_drain(tsd, quarantine, upper_bound); } /* Grow the quarantine ring buffer if it's full. */ if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) - quarantine = quarantine_grow(quarantine); + quarantine = quarantine_grow(tsd, quarantine); /* quarantine_grow() must free a slot if it fails to grow. */ assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); /* Append ptr if its size doesn't exceed the quarantine size. */ @@ -154,46 +139,22 @@ quarantine(void *ptr) } } else { assert(quarantine->curbytes == 0); - idalloc(ptr); + idalloc(tsd, ptr); } } void -quarantine_cleanup(void *arg) +quarantine_cleanup(tsd_t *tsd) { - quarantine_t *quarantine = *(quarantine_t **)arg; + quarantine_t *quarantine; - if (quarantine == QUARANTINE_STATE_REINCARNATED) { - /* - * Another destructor deallocated memory after this destructor - * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY - * in order to receive another callback. - */ - quarantine = QUARANTINE_STATE_PURGATORY; - quarantine_tsd_set(&quarantine); - } else if (quarantine == QUARANTINE_STATE_PURGATORY) { - /* - * The previous time this destructor was called, we set the key - * to QUARANTINE_STATE_PURGATORY so that other destructors - * wouldn't cause re-creation of the quarantine. This time, do - * nothing, so that the destructor will not be called again. - */ - } else if (quarantine != NULL) { - quarantine_drain(quarantine, 0); - idalloc(quarantine); - quarantine = QUARANTINE_STATE_PURGATORY; - quarantine_tsd_set(&quarantine); + if (!config_fill) + return; + + quarantine = tsd_quarantine_get(tsd); + if (quarantine != NULL) { + quarantine_drain(tsd, quarantine, 0); + idalloc(tsd, quarantine); + tsd_quarantine_set(tsd, NULL); } } - -bool -quarantine_boot(void) -{ - - cassert(config_fill); - - if (quarantine_tsd_boot()) - return (true); - - return (false); -} diff --git a/src/rtree.c b/src/rtree.c index 87b0b154..2ff93dbe 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -9,8 +9,10 @@ rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc) assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3)); - bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1; - bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1; + bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void + *)))) - 1; + bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / + sizeof(uint8_t)))) - 1; if (bits > bits_in_leaf) { height = 1 + (bits - bits_in_leaf) / bits_per_level; if ((height-1) * bits_per_level + bits_in_leaf != bits) diff --git a/src/tcache.c b/src/tcache.c index f86a46e6..bb4c3cc0 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -4,9 +4,6 @@ /******************************************************************************/ /* Data. */ -malloc_tsd_data(, tcache, tcache_t *, NULL) -malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default) - bool opt_tcache = true; ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; @@ -262,43 +259,14 @@ tcache_arena_dissociate(tcache_t *tcache) } tcache_t * -tcache_get_hard(tcache_t *tcache, bool create) +tcache_get_hard(tsd_t *tsd) { - if (tcache == NULL) { - if (create == false) { - /* - * Creating a tcache here would cause - * allocation as a side effect of free(). - * Ordinarily that would be okay since - * tcache_create() failure is a soft failure - * that doesn't propagate. However, if TLS - * data are freed via free() as in glibc, - * subtle corruption could result from setting - * a TLS variable after its backing memory is - * freed. - */ - return (NULL); - } - if (tcache_enabled_get() == false) { - tcache_enabled_set(false); /* Memoize. */ - return (NULL); - } - return (tcache_create(choose_arena(NULL))); - } - if (tcache == TCACHE_STATE_PURGATORY) { - /* - * Make a note that an allocator function was called - * after tcache_thread_cleanup() was called. - */ - tcache = TCACHE_STATE_REINCARNATED; - tcache_tsd_set(&tcache); + if (tcache_enabled_get() == false) { + tcache_enabled_set(false); /* Memoize. */ return (NULL); } - if (tcache == TCACHE_STATE_REINCARNATED) - return (NULL); - not_reached(); - return (NULL); + return (tcache_create(choose_arena(tsd, NULL))); } tcache_t * @@ -328,7 +296,7 @@ tcache_create(arena_t *arena) else if (size <= tcache_maxclass) tcache = (tcache_t *)arena_malloc_large(arena, size, true); else - tcache = (tcache_t *)icalloct(size, false, arena); + tcache = (tcache_t *)icalloct(NULL, size, false, arena); if (tcache == NULL) return (NULL); @@ -343,13 +311,11 @@ tcache_create(arena_t *arena) stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); } - tcache_tsd_set(&tcache); - return (tcache); } -void -tcache_destroy(tcache_t *tcache) +static void +tcache_destroy(tsd_t *tsd, tcache_t *tcache) { unsigned i; size_t tcache_size; @@ -403,39 +369,30 @@ tcache_destroy(tcache_t *tcache) arena_dalloc_large(arena, chunk, tcache); } else - idalloct(tcache, false); + idalloct(tsd, tcache, false); } void -tcache_thread_cleanup(void *arg) +tcache_cleanup(tsd_t *tsd) { - tcache_t *tcache = *(tcache_t **)arg; + tcache_t *tcache; - if (tcache == TCACHE_STATE_DISABLED) { - /* Do nothing. */ - } else if (tcache == TCACHE_STATE_REINCARNATED) { - /* - * Another destructor called an allocator function after this - * destructor was called. Reset tcache to - * TCACHE_STATE_PURGATORY in order to receive another callback. - */ - tcache = TCACHE_STATE_PURGATORY; - tcache_tsd_set(&tcache); - } else if (tcache == TCACHE_STATE_PURGATORY) { - /* - * The previous time this destructor was called, we set the key - * to TCACHE_STATE_PURGATORY so that other destructors wouldn't - * cause re-creation of the tcache. This time, do nothing, so - * that the destructor will not be called again. - */ - } else if (tcache != NULL) { - assert(tcache != TCACHE_STATE_PURGATORY); - tcache_destroy(tcache); - tcache = TCACHE_STATE_PURGATORY; - tcache_tsd_set(&tcache); + if (!config_tcache) + return; + + if ((tcache = tsd_tcache_get(tsd)) != NULL) { + tcache_destroy(tsd, tcache); + tsd_tcache_set(tsd, NULL); } } +void +tcache_enabled_cleanup(tsd_t *tsd) +{ + + /* Do nothing. */ +} + /* Caller must own arena->lock. */ void tcache_stats_merge(tcache_t *tcache, arena_t *arena) @@ -464,7 +421,7 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena) } bool -tcache_boot0(void) +tcache_boot(void) { unsigned i; @@ -504,13 +461,3 @@ tcache_boot0(void) return (false); } - -bool -tcache_boot1(void) -{ - - if (tcache_tsd_boot() || tcache_enabled_tsd_boot()) - return (true); - - return (false); -} diff --git a/src/tsd.c b/src/tsd.c index 700caabf..27a70ee8 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -7,6 +7,8 @@ static unsigned ncleanups; static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; +malloc_tsd_data(, , tsd_t, TSD_INITIALIZER) + /******************************************************************************/ void * @@ -14,14 +16,15 @@ malloc_tsd_malloc(size_t size) { /* Avoid choose_arena() in order to dodge bootstrapping issues. */ - return (arena_malloc(arenas[0], size, false, false)); + return (arena_malloc(NULL, arenas[0], CACHELINE_CEILING(size), false, + false)); } void malloc_tsd_dalloc(void *wrapper) { - idalloct(wrapper, false); + idalloct(NULL, wrapper, false); } void @@ -67,10 +70,54 @@ malloc_tsd_cleanup_register(bool (*f)(void)) } void +tsd_cleanup(void *arg) +{ + tsd_t *tsd = (tsd_t *)arg; + + if (tsd == NULL) { + /* OOM during re-initialization. */ + return; + } + + switch (tsd->state) { + case tsd_state_nominal: +#define O(n, t) \ + n##_cleanup(tsd); +MALLOC_TSD +#undef O + tsd->state = tsd_state_purgatory; + tsd_set(tsd); + break; + case tsd_state_purgatory: + /* + * The previous time this destructor was called, we set the + * state to tsd_state_purgatory so that other destructors + * wouldn't cause re-creation of the tsd. This time, do + * nothing, and do not request another callback. + */ + break; + case tsd_state_reincarnated: + /* + * Another destructor deallocated memory after this destructor + * was called. Reset state to tsd_state_purgatory and request + * another callback. + */ + tsd->state = tsd_state_purgatory; + tsd_set(tsd); + break; + default: + not_reached(); + } +} + +bool malloc_tsd_boot(void) { ncleanups = 0; + if (tsd_boot()) + return (true); + return (false); } #ifdef _WIN32 diff --git a/test/unit/ckh.c b/test/unit/ckh.c index b214c279..148b81e7 100644 --- a/test/unit/ckh.c +++ b/test/unit/ckh.c @@ -2,20 +2,25 @@ TEST_BEGIN(test_new_delete) { + tsd_t *tsd; ckh_t ckh; - assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp), - "Unexpected ckh_new() error"); - ckh_delete(&ckh); + tsd = tsd_tryget(); + assert_ptr_not_null(tsd, "Unexpected tsd failure"); - assert_false(ckh_new(&ckh, 3, ckh_pointer_hash, ckh_pointer_keycomp), + assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp), "Unexpected ckh_new() error"); - ckh_delete(&ckh); + ckh_delete(tsd, &ckh); + + assert_false(ckh_new(tsd, &ckh, 3, ckh_pointer_hash, + ckh_pointer_keycomp), "Unexpected ckh_new() error"); + ckh_delete(tsd, &ckh); } TEST_END TEST_BEGIN(test_count_insert_search_remove) { + tsd_t *tsd; ckh_t ckh; const char *strs[] = { "a string", @@ -26,7 +31,10 @@ TEST_BEGIN(test_count_insert_search_remove) const char *missing = "A string not in the hash table."; size_t i; - assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp), + tsd = tsd_tryget(); + assert_ptr_not_null(tsd, "Unexpected tsd failure"); + + assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp), "Unexpected ckh_new() error"); assert_zu_eq(ckh_count(&ckh), 0, "ckh_count() should return %zu, but it returned %zu", ZU(0), @@ -34,7 +42,7 @@ TEST_BEGIN(test_count_insert_search_remove) /* Insert. */ for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) { - ckh_insert(&ckh, strs[i], strs[i]); + ckh_insert(tsd, &ckh, strs[i], strs[i]); assert_zu_eq(ckh_count(&ckh), i+1, "ckh_count() should return %zu, but it returned %zu", i+1, ckh_count(&ckh)); @@ -79,7 +87,7 @@ TEST_BEGIN(test_count_insert_search_remove) vp = (i & 2) ? &v.p : NULL; k.p = NULL; v.p = NULL; - assert_false(ckh_remove(&ckh, strs[i], kp, vp), + assert_false(ckh_remove(tsd, &ckh, strs[i], kp, vp), "Unexpected ckh_remove() error"); ks = (i & 1) ? strs[i] : (const char *)NULL; @@ -95,20 +103,24 @@ TEST_BEGIN(test_count_insert_search_remove) ckh_count(&ckh)); } - ckh_delete(&ckh); + ckh_delete(tsd, &ckh); } TEST_END TEST_BEGIN(test_insert_iter_remove) { #define NITEMS ZU(1000) + tsd_t *tsd; ckh_t ckh; void **p[NITEMS]; void *q, *r; size_t i; - assert_false(ckh_new(&ckh, 2, ckh_pointer_hash, ckh_pointer_keycomp), - "Unexpected ckh_new() error"); + tsd = tsd_tryget(); + assert_ptr_not_null(tsd, "Unexpected tsd failure"); + + assert_false(ckh_new(tsd, &ckh, 2, ckh_pointer_hash, + ckh_pointer_keycomp), "Unexpected ckh_new() error"); for (i = 0; i < NITEMS; i++) { p[i] = mallocx(i+1, 0); @@ -119,7 +131,7 @@ TEST_BEGIN(test_insert_iter_remove) size_t j; for (j = i; j < NITEMS; j++) { - assert_false(ckh_insert(&ckh, p[j], p[j]), + assert_false(ckh_insert(tsd, &ckh, p[j], p[j]), "Unexpected ckh_insert() failure"); assert_false(ckh_search(&ckh, p[j], &q, &r), "Unexpected ckh_search() failure"); @@ -134,13 +146,13 @@ TEST_BEGIN(test_insert_iter_remove) for (j = i + 1; j < NITEMS; j++) { assert_false(ckh_search(&ckh, p[j], NULL, NULL), "Unexpected ckh_search() failure"); - assert_false(ckh_remove(&ckh, p[j], &q, &r), + assert_false(ckh_remove(tsd, &ckh, p[j], &q, &r), "Unexpected ckh_remove() failure"); assert_ptr_eq(p[j], q, "Key pointer mismatch"); assert_ptr_eq(p[j], r, "Value pointer mismatch"); assert_true(ckh_search(&ckh, p[j], NULL, NULL), "Unexpected ckh_search() success"); - assert_true(ckh_remove(&ckh, p[j], &q, &r), + assert_true(ckh_remove(tsd, &ckh, p[j], &q, &r), "Unexpected ckh_remove() success"); } @@ -176,13 +188,13 @@ TEST_BEGIN(test_insert_iter_remove) for (i = 0; i < NITEMS; i++) { assert_false(ckh_search(&ckh, p[i], NULL, NULL), "Unexpected ckh_search() failure"); - assert_false(ckh_remove(&ckh, p[i], &q, &r), + assert_false(ckh_remove(tsd, &ckh, p[i], &q, &r), "Unexpected ckh_remove() failure"); assert_ptr_eq(p[i], q, "Key pointer mismatch"); assert_ptr_eq(p[i], r, "Value pointer mismatch"); assert_true(ckh_search(&ckh, p[i], NULL, NULL), "Unexpected ckh_search() success"); - assert_true(ckh_remove(&ckh, p[i], &q, &r), + assert_true(ckh_remove(tsd, &ckh, p[i], &q, &r), "Unexpected ckh_remove() success"); dallocx(p[i], 0); } @@ -190,7 +202,7 @@ TEST_BEGIN(test_insert_iter_remove) assert_zu_eq(ckh_count(&ckh), 0, "ckh_count() should return %zu, but it returned %zu", ZU(0), ckh_count(&ckh)); - ckh_delete(&ckh); + ckh_delete(tsd, &ckh); #undef NITEMS } TEST_END diff --git a/test/unit/rtree.c b/test/unit/rtree.c index 5463055f..77a947d6 100644 --- a/test/unit/rtree.c +++ b/test/unit/rtree.c @@ -5,7 +5,7 @@ TEST_BEGIN(test_rtree_get_empty) unsigned i; for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { - rtree_t *rtree = rtree_new(i, imalloc, idalloc); + rtree_t *rtree = rtree_new(i, malloc, free); assert_u_eq(rtree_get(rtree, 0), 0, "rtree_get() should return NULL for empty tree"); rtree_delete(rtree); @@ -18,7 +18,7 @@ TEST_BEGIN(test_rtree_extrema) unsigned i; for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { - rtree_t *rtree = rtree_new(i, imalloc, idalloc); + rtree_t *rtree = rtree_new(i, malloc, free); rtree_set(rtree, 0, 1); assert_u_eq(rtree_get(rtree, 0), 1, @@ -40,7 +40,7 @@ TEST_BEGIN(test_rtree_bits) for (i = 1; i < (sizeof(uintptr_t) << 3); i++) { uintptr_t keys[] = {0, 1, (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1}; - rtree_t *rtree = rtree_new(i, imalloc, idalloc); + rtree_t *rtree = rtree_new(i, malloc, free); for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { rtree_set(rtree, keys[j], 1); @@ -73,7 +73,7 @@ TEST_BEGIN(test_rtree_random) sfmt = init_gen_rand(SEED); for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { - rtree_t *rtree = rtree_new(i, imalloc, idalloc); + rtree_t *rtree = rtree_new(i, malloc, free); uintptr_t keys[NSET]; unsigned j; diff --git a/test/unit/tsd.c b/test/unit/tsd.c index f421c1a3..391a7807 100644 --- a/test/unit/tsd.c +++ b/test/unit/tsd.c @@ -16,11 +16,11 @@ data_cleanup(void *arg) data_cleanup_executed = true; } -malloc_tsd_protos(, data, data_t) -malloc_tsd_externs(data, data_t) +malloc_tsd_protos(, data_, data_t) +malloc_tsd_externs(data_, data_t) #define DATA_INIT 0x12345678 -malloc_tsd_data(, data, data_t, DATA_INIT) -malloc_tsd_funcs(, data, data_t, DATA_INIT, data_cleanup) +malloc_tsd_data(, data_, data_t, DATA_INIT) +malloc_tsd_funcs(, data_, data_t, DATA_INIT, data_cleanup) static void * thd_start(void *arg) From eb5376ab9e61d96daa0d1f03b4474baf5232478f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 23 Sep 2014 09:21:49 -0700 Subject: [PATCH 113/352] Add instructions for installing from non-packaged sources. --- INSTALL | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/INSTALL b/INSTALL index 6c46100e..9af23369 100644 --- a/INSTALL +++ b/INSTALL @@ -1,10 +1,23 @@ -Building and installing jemalloc can be as simple as typing the following while -in the root directory of the source tree: +Building and installing a packaged release of jemalloc can be as simple as +typing the following while in the root directory of the source tree: ./configure make make install +If building from unpackaged developer sources, the simplest command sequence +that might work is: + + ./autogen.sh + make dist + make + make install + +Note that documentation is not built by the default target because doing so +would create a dependency on xsltproc in packaged releases, hence the +requirement to either run 'make dist' or avoid installing docs via the various +install_* targets documented below. + === Advanced configuration ===================================================== The 'configure' script supports numerous options that allow control of which From 70bdee07d9e3942580e576b94010108c342d609d Mon Sep 17 00:00:00 2001 From: Dave Rigby Date: Mon, 22 Sep 2014 15:53:16 +0100 Subject: [PATCH 114/352] autoconf: Support cygwin in addition to mingw --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 2d5b56a4..ab4bcc39 100644 --- a/configure.ac +++ b/configure.ac @@ -330,7 +330,7 @@ case "${host}" in fi abi="xcoff" ;; - *-*-mingw*) + *-*-mingw* | *-*-cygwin*) abi="pecoff" force_tls="0" RPATH="" From 112704cfbfacfc9cecdfb732741df47eb4133902 Mon Sep 17 00:00:00 2001 From: Dave Rigby Date: Mon, 22 Sep 2014 15:54:33 +0100 Subject: [PATCH 115/352] Use MSVC intrinsics for lg_floor When using MSVC make use of its intrinsic functions (supported on x86, amd64 & ARM) for lg_floor. --- include/jemalloc/internal/util.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index cc7806d0..5af68329 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -176,6 +176,21 @@ lg_floor(size_t x) ); return (ret); } +#elif (defined(_MSC_VER)) +JEMALLOC_INLINE size_t +lg_floor(size_t x) +{ + unsigned long ret; + +#if (LG_SIZEOF_PTR == 3) + _BitScanReverse64(&ret, x); +#elif (LG_SIZEOF_PTR == 2) + _BitScanReverse(&ret, x); +#else +# error "Unsupported type sizes for lg_floor()" +#endif + return (ret); +} #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) JEMALLOC_INLINE size_t lg_floor(size_t x) From 6ef80d68f092caf3b3802a73b8d716057b41864c Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 24 Sep 2014 22:14:21 -0700 Subject: [PATCH 116/352] Fix profile dumping race. Fix a race that caused a non-critical assertion failure. To trigger the race, a thread had to be part way through initializing a new sample, such that it was discoverable by the dumping thread, but not yet linked into its gctx by the time a later dump phase would normally have reset its state to 'nominal'. Additionally, lock access to the state field during modification to transition to the dumping state. It's not apparent that this oversight could have caused an actual problem due to outer locking that protects the dumping machinery, but the added locking pedantically follows the stated locking protocol for the state field. --- include/jemalloc/internal/prof.h | 1 + src/prof.c | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index b8a8b419..3872c7ae 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -79,6 +79,7 @@ struct prof_cnt_s { }; typedef enum { + prof_tctx_state_initializing, prof_tctx_state_nominal, prof_tctx_state_dumping, prof_tctx_state_purgatory /* Dumper must finish destroying. */ diff --git a/src/prof.c b/src/prof.c index dd84f533..9f10b533 100644 --- a/src/prof.c +++ b/src/prof.c @@ -717,7 +717,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); ret.p->gctx = gctx; ret.p->prepared = true; - ret.p->state = prof_tctx_state_nominal; + ret.p->state = prof_tctx_state_initializing; malloc_mutex_lock(tdata->lock); error = ckh_insert(tsd, &tdata->bt2tctx, btkey, ret.v); malloc_mutex_unlock(tdata->lock); @@ -728,6 +728,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) return (NULL); } malloc_mutex_lock(gctx->lock); + ret.p->state = prof_tctx_state_nominal; tctx_tree_insert(&gctx->tctxs, ret.p); gctx->nlimbo--; malloc_mutex_unlock(gctx->lock); @@ -925,8 +926,15 @@ static void prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata) { + malloc_mutex_lock(tctx->gctx->lock); + if (tctx->state == prof_tctx_state_initializing) { + malloc_mutex_unlock(tctx->gctx->lock); + return; + } assert(tctx->state == prof_tctx_state_nominal); tctx->state = prof_tctx_state_dumping; + malloc_mutex_unlock(tctx->gctx->lock); + memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; From f97e5ac4ec8a5ae7ed74829e6c1bf6ce814947f5 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 28 Sep 2014 14:43:11 -0700 Subject: [PATCH 117/352] Implement compile-time bitmap size computation. --- include/jemalloc/internal/bitmap.h | 46 ++++++++++++++++++++++++++++++ src/bitmap.c | 18 ++---------- test/unit/bitmap.c | 16 ++++------- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/include/jemalloc/internal/bitmap.h b/include/jemalloc/internal/bitmap.h index 6db4ab70..4ca40ffd 100644 --- a/include/jemalloc/internal/bitmap.h +++ b/include/jemalloc/internal/bitmap.h @@ -3,6 +3,7 @@ /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ #define LG_BITMAP_MAXBITS LG_RUN_MAXREGS +#define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) typedef struct bitmap_level_s bitmap_level_t; typedef struct bitmap_info_s bitmap_info_t; @@ -14,6 +15,51 @@ typedef unsigned long bitmap_t; #define BITMAP_GROUP_NBITS (ZU(1) << LG_BITMAP_GROUP_NBITS) #define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1) +/* Number of groups required to store a given number of bits. */ +#define BITMAP_BITS2GROUPS(nbits) \ + ((nbits + BITMAP_GROUP_NBITS_MASK) >> LG_BITMAP_GROUP_NBITS) + +/* + * Number of groups required at a particular level for a given number of bits. + */ +#define BITMAP_GROUPS_L0(nbits) \ + BITMAP_BITS2GROUPS(nbits) +#define BITMAP_GROUPS_L1(nbits) \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(nbits)) +#define BITMAP_GROUPS_L2(nbits) \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS((nbits)))) +#define BITMAP_GROUPS_L3(nbits) \ + BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS(BITMAP_BITS2GROUPS( \ + BITMAP_BITS2GROUPS((nbits))))) + +/* + * Assuming the number of levels, number of groups required for a given number + * of bits. + */ +#define BITMAP_GROUPS_1_LEVEL(nbits) \ + BITMAP_GROUPS_L0(nbits) +#define BITMAP_GROUPS_2_LEVEL(nbits) \ + (BITMAP_GROUPS_1_LEVEL(nbits) + BITMAP_GROUPS_L1(nbits)) +#define BITMAP_GROUPS_3_LEVEL(nbits) \ + (BITMAP_GROUPS_2_LEVEL(nbits) + BITMAP_GROUPS_L2(nbits)) +#define BITMAP_GROUPS_4_LEVEL(nbits) \ + (BITMAP_GROUPS_3_LEVEL(nbits) + BITMAP_GROUPS_L3(nbits)) + +/* + * Maximum number of groups required to support LG_BITMAP_MAXBITS. + */ +#if LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_1_LEVEL(BITMAP_MAXBITS) +#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 2 +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_2_LEVEL(BITMAP_MAXBITS) +#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 3 +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_3_LEVEL(BITMAP_MAXBITS) +#elif LG_BITMAP_MAXBITS <= LG_BITMAP_GROUP_NBITS * 4 +# define BITMAP_GROUPS_MAX BITMAP_GROUPS_4_LEVEL(BITMAP_MAXBITS) +#else +# error "Unsupported bitmap size" +#endif + /* Maximum number of levels possible. */ #define BITMAP_MAX_LEVELS \ (LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) \ diff --git a/src/bitmap.c b/src/bitmap.c index e2bd907d..c733372b 100644 --- a/src/bitmap.c +++ b/src/bitmap.c @@ -2,19 +2,6 @@ #include "jemalloc/internal/jemalloc_internal.h" /******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static size_t bits2groups(size_t nbits); - -/******************************************************************************/ - -static size_t -bits2groups(size_t nbits) -{ - - return ((nbits >> LG_BITMAP_GROUP_NBITS) + - !!(nbits & BITMAP_GROUP_NBITS_MASK)); -} void bitmap_info_init(bitmap_info_t *binfo, size_t nbits) @@ -31,15 +18,16 @@ bitmap_info_init(bitmap_info_t *binfo, size_t nbits) * that requires only one group. */ binfo->levels[0].group_offset = 0; - group_count = bits2groups(nbits); + group_count = BITMAP_BITS2GROUPS(nbits); for (i = 1; group_count > 1; i++) { assert(i < BITMAP_MAX_LEVELS); binfo->levels[i].group_offset = binfo->levels[i-1].group_offset + group_count; - group_count = bits2groups(group_count); + group_count = BITMAP_BITS2GROUPS(group_count); } binfo->levels[i].group_offset = binfo->levels[i-1].group_offset + group_count; + assert(binfo->levels[i].group_offset <= BITMAP_GROUPS_MAX); binfo->nlevels = i; binfo->nbits = nbits; } diff --git a/test/unit/bitmap.c b/test/unit/bitmap.c index 8086b888..4ea94f85 100644 --- a/test/unit/bitmap.c +++ b/test/unit/bitmap.c @@ -1,17 +1,11 @@ #include "test/jemalloc_test.h" -#if (LG_BITMAP_MAXBITS > 12) -# define MAXBITS 4500 -#else -# define MAXBITS (1U << LG_BITMAP_MAXBITS) -#endif - TEST_BEGIN(test_bitmap_size) { size_t i, prev_size; prev_size = 0; - for (i = 1; i <= MAXBITS; i++) { + for (i = 1; i <= BITMAP_MAXBITS; i++) { size_t size = bitmap_size(i); assert_true(size >= prev_size, "Bitmap size is smaller than expected"); @@ -24,7 +18,7 @@ TEST_BEGIN(test_bitmap_init) { size_t i; - for (i = 1; i <= MAXBITS; i++) { + for (i = 1; i <= BITMAP_MAXBITS; i++) { bitmap_info_t binfo; bitmap_info_init(&binfo, i); { @@ -47,7 +41,7 @@ TEST_BEGIN(test_bitmap_set) { size_t i; - for (i = 1; i <= MAXBITS; i++) { + for (i = 1; i <= BITMAP_MAXBITS; i++) { bitmap_info_t binfo; bitmap_info_init(&binfo, i); { @@ -70,7 +64,7 @@ TEST_BEGIN(test_bitmap_unset) { size_t i; - for (i = 1; i <= MAXBITS; i++) { + for (i = 1; i <= BITMAP_MAXBITS; i++) { bitmap_info_t binfo; bitmap_info_init(&binfo, i); { @@ -99,7 +93,7 @@ TEST_BEGIN(test_bitmap_sfu) { size_t i; - for (i = 1; i <= MAXBITS; i++) { + for (i = 1; i <= BITMAP_MAXBITS; i++) { bitmap_info_t binfo; bitmap_info_init(&binfo, i); { From 0c5dd03e889d0269170b5db9fa872738d906eb78 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 29 Sep 2014 01:31:39 -0700 Subject: [PATCH 118/352] Move small run metadata into the arena chunk header. Move small run metadata into the arena chunk header, with multiple expected benefits: - Lower run fragmentation due to reduced run sizes; runs are more likely to completely drain when there are fewer total regions. - Improved cache behavior. Prior to this change, run headers were always page-aligned, which put extra pressure on some CPU cache sets. The degree to which this was a problem was hardware dependent, but it likely hurt some even for the most advanced modern hardware. - Buffer overruns/underruns are less likely to corrupt allocator metadata. - Size classes between 4 KiB and 16 KiB become reasonable to support without any special handling, and the runs are small enough that dirty unused pages aren't a significant concern. --- include/jemalloc/internal/arena.h | 144 ++++---- include/jemalloc/internal/private_symbols.txt | 3 + src/arena.c | 345 ++++++++---------- 3 files changed, 232 insertions(+), 260 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index f1a12057..48fd2055 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -1,30 +1,8 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -/* - * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized - * as small as possible such that this setting is still honored, without - * violating other constraints. The goal is to make runs as small as possible - * without exceeding a per run external fragmentation threshold. - * - * We use binary fixed point math for overhead computations, where the binary - * point is implicitly RUN_BFP bits to the left. - * - * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be - * honored for some/all object sizes, since when heap profiling is enabled - * there is one pointer of header overhead per object (plus a constant). This - * constraint is relaxed (ignored) for runs that are so small that the - * per-region overhead is greater than: - * - * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) - */ -#define RUN_BFP 12 -/* \/ Implicit binary fixed point. */ -#define RUN_MAX_OVRHD 0x0000003dU -#define RUN_MAX_OVRHD_RELAX 0x00001800U - /* Maximum number of regions in one run. */ -#define LG_RUN_MAXREGS 11 +#define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) #define RUN_MAXREGS (1U << LG_RUN_MAXREGS) /* @@ -43,10 +21,10 @@ */ #define LG_DIRTY_MULT_DEFAULT 3 +typedef struct arena_run_s arena_run_t; typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; typedef struct arena_chunk_s arena_chunk_t; -typedef struct arena_run_s arena_run_t; typedef struct arena_bin_info_s arena_bin_info_t; typedef struct arena_bin_s arena_bin_t; typedef struct arena_s arena_t; @@ -55,6 +33,20 @@ typedef struct arena_s arena_t; /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +struct arena_run_s { + /* Bin this run is associated with. */ + arena_bin_t *bin; + + /* Index of next region that has never been allocated, or nregs. */ + uint32_t nextind; + + /* Number of free regions in run. */ + unsigned nfree; + + /* Per region allocated/deallocated bitmap. */ + bitmap_t bitmap[BITMAP_GROUPS_MAX]; +}; + /* Each element of the chunk map corresponds to one page within the chunk. */ struct arena_chunk_map_bits_s { /* @@ -130,15 +122,6 @@ struct arena_chunk_map_bits_s { * chunk header in order to improve cache locality. */ struct arena_chunk_map_misc_s { -#ifndef JEMALLOC_PROF - /* - * Overlay prof_tctx in order to allow it to be referenced by dead code. - * Such antics aren't warranted for per arena data structures, but - * chunk map overhead accounts for a percentage of memory, rather than - * being just a fixed cost. - */ - union { -#endif /* * Linkage for run trees. There are two disjoint uses: * @@ -146,16 +129,18 @@ struct arena_chunk_map_misc_s { * 2) arena_run_t conceptually uses this linkage for in-use non-full * runs, rather than directly embedding linkage. */ - rb_node(arena_chunk_map_misc_t) rb_link; + rb_node(arena_chunk_map_misc_t) rb_link; - /* Profile counters, used for large object runs. */ - prof_tctx_t *prof_tctx; -#ifndef JEMALLOC_PROF - }; /* union { ... }; */ -#endif + union { + /* Linkage for list of dirty runs. */ + ql_elm(arena_chunk_map_misc_t) dr_link; - /* Linkage for list of dirty runs. */ - ql_elm(arena_chunk_map_misc_t) dr_link; + /* Profile counters, used for large object runs. */ + prof_tctx_t *prof_tctx; + + /* Small region run metadata. */ + arena_run_t run; + }; }; typedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t; typedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; @@ -175,17 +160,6 @@ struct arena_chunk_s { arena_chunk_map_bits_t map_bits[1]; /* Dynamically sized. */ }; -struct arena_run_s { - /* Bin this run is associated with. */ - arena_bin_t *bin; - - /* Index of next region that has never been allocated, or nregs. */ - uint32_t nextind; - - /* Number of free regions in run. */ - unsigned nfree; -}; - /* * Read-only information associated with each element of arena_t's bins array * is stored separately, partly to reduce memory usage (only one copy, rather @@ -194,10 +168,7 @@ struct arena_run_s { * Each run has the following layout: * * /--------------------\ - * | arena_run_t header | - * | ... | - * bitmap_offset | bitmap | - * | ... | + * | pad? | * |--------------------| * | redzone | * reg0_offset | region 0 | @@ -238,12 +209,6 @@ struct arena_bin_info_s { /* Total number of regions in a run for this bin's size class. */ uint32_t nregs; - /* - * Offset of first bitmap_t element in a run header for this bin's size - * class. - */ - uint32_t bitmap_offset; - /* * Metadata used to manipulate bitmaps for runs associated with this * bin. @@ -451,6 +416,9 @@ arena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, size_t pageind); arena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm); +void *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); +arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbitsp_read(size_t *mapbitsp); size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); @@ -659,6 +627,40 @@ arena_miscelm_get(arena_chunk_t *chunk, size_t pageind) (uintptr_t)map_misc_offset) + pageind-map_bias); } +JEMALLOC_ALWAYS_INLINE size_t +arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm) +{ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + size_t pageind = ((uintptr_t)miscelm - ((uintptr_t)chunk + + map_misc_offset)) / sizeof(arena_chunk_map_misc_t) + map_bias; + + assert(pageind >= map_bias); + assert(pageind < chunk_npages); + + return (pageind); +} + +JEMALLOC_ALWAYS_INLINE void * +arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm) +{ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); + + return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); +} + +JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * +arena_run_to_miscelm(arena_run_t *run) +{ + arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t + *)((uintptr_t)run - offsetof(arena_chunk_map_misc_t, run)); + + assert(arena_miscelm_to_pageind(miscelm) >= map_bias); + assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); + + return (miscelm); +} + JEMALLOC_ALWAYS_INLINE size_t * arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) { @@ -903,10 +905,13 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) arena_t *arena; size_t pageind; size_t actual_mapbits; + size_t rpages_ind; arena_run_t *run; arena_bin_t *bin; size_t actual_binind; arena_bin_info_t *bin_info; + arena_chunk_map_misc_t *miscelm; + void *rpages; assert(binind != BININD_INVALID); assert(binind < NBINS); @@ -917,13 +922,16 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) assert(mapbits == actual_mapbits); assert(arena_mapbits_large_get(chunk, pageind) == 0); assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (actual_mapbits >> LG_PAGE)) << LG_PAGE)); + rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, + pageind); + miscelm = arena_miscelm_get(chunk, rpages_ind); + run = &miscelm->run; bin = run->bin; actual_binind = bin - arena->bins; assert(binind == actual_binind); bin_info = &arena_bin_info[actual_binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + + rpages = arena_miscelm_to_rpages(miscelm); + assert(((uintptr_t)ptr - ((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval == 0); } @@ -946,19 +954,21 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) { unsigned shift, diff, regind; size_t interval; + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); + void *rpages = arena_miscelm_to_rpages(miscelm); /* * Freeing a pointer lower than region zero can cause assertion * failure. */ - assert((uintptr_t)ptr >= (uintptr_t)run + + assert((uintptr_t)ptr >= (uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset); /* * Avoid doing division with a variable divisor if possible. Using * actual division here can reduce allocator throughput by over 20%! */ - diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - + diff = (unsigned)((uintptr_t)ptr - (uintptr_t)rpages - bin_info->reg0_offset); /* Rescale (factor powers of 2 out of the numerator and denominator). */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 84d48d19..5ac82f59 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -42,6 +42,8 @@ arena_mapbitsp_read arena_mapbitsp_write arena_maxclass arena_miscelm_get +arena_miscelm_to_pageind +arena_miscelm_to_rpages arena_new arena_palloc arena_postfork_child @@ -61,6 +63,7 @@ arena_ralloc_junk_large arena_ralloc_no_move arena_redzone_corruption arena_run_regind +arena_run_to_miscelm arena_salloc arena_sdalloc arena_stats_merge diff --git a/src/arena.c b/src/arena.c index 40da9f47..ef391b16 100644 --- a/src/arena.c +++ b/src/arena.c @@ -60,15 +60,6 @@ static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, /******************************************************************************/ -JEMALLOC_INLINE_C size_t -arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm) -{ - size_t offset = CHUNK_ADDR2OFFSET(miscelm); - - return ((offset - map_misc_offset) / sizeof(arena_chunk_map_misc_t) + - map_bias); -} - JEMALLOC_INLINE_C size_t arena_miscelm_to_bits(arena_chunk_map_misc_t *miscelm) { @@ -183,14 +174,16 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) { void *ret; unsigned regind; - bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + - (uintptr_t)bin_info->bitmap_offset); + arena_chunk_map_misc_t *miscelm; + void *rpages; assert(run->nfree > 0); - assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); + assert(bitmap_full(run->bitmap, &bin_info->bitmap_info) == false); - regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); - ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + + regind = bitmap_sfu(run->bitmap, &bin_info->bitmap_info); + miscelm = arena_run_to_miscelm(run); + rpages = arena_miscelm_to_rpages(miscelm); + ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(bin_info->reg_interval * regind)); run->nfree--; if (regind == run->nextind) @@ -208,20 +201,20 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) size_t binind = arena_ptr_small_binind_get(ptr, mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind = arena_run_regind(run, bin_info, ptr); - bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + - (uintptr_t)bin_info->bitmap_offset); assert(run->nfree < bin_info->nregs); /* Freeing an interior pointer can cause assertion failure. */ - assert(((uintptr_t)ptr - ((uintptr_t)run + + assert(((uintptr_t)ptr - + ((uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_interval == 0); - assert((uintptr_t)ptr >= (uintptr_t)run + + assert((uintptr_t)ptr >= + (uintptr_t)arena_miscelm_to_rpages(arena_run_to_miscelm(run)) + (uintptr_t)bin_info->reg0_offset); /* Freeing an unallocated pointer can cause assertion failure. */ - assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); + assert(bitmap_get(run->bitmap, &bin_info->bitmap_info, regind)); - bitmap_unset(bitmap, &bin_info->bitmap_info, regind); + bitmap_unset(run->bitmap, &bin_info->bitmap_info, regind); run->nfree++; } @@ -316,10 +309,12 @@ arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, bool remove, bool zero) { arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; size_t flag_dirty, run_ind, need_pages, i; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + miscelm = arena_run_to_miscelm(run); + run_ind = arena_miscelm_to_pageind(miscelm); flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); need_pages = (size >> LG_PAGE); assert(need_pages > 0); @@ -383,12 +378,14 @@ arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, size_t binind) { arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; size_t flag_dirty, run_ind, need_pages, i; assert(binind != BININD_INVALID); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + miscelm = arena_run_to_miscelm(run); + run_ind = arena_miscelm_to_pageind(miscelm); flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); need_pages = (size >> LG_PAGE); assert(need_pages > 0); @@ -401,11 +398,6 @@ arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, * clean pages. */ arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); - /* - * The first page will always be dirtied during small run - * initialization, so a validation failure here would not actually - * cause an observable failure. - */ if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, run_ind) == 0) arena_run_page_validate_zeroed(chunk, run_ind); @@ -643,19 +635,14 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) static arena_run_t * arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { - arena_run_t *run; arena_chunk_map_misc_t *miscelm; arena_chunk_map_misc_t *key; key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (miscelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(miscelm); - size_t pageind = arena_miscelm_to_pageind(miscelm); - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - LG_PAGE)); - arena_run_split_large(arena, run, size, zero); + arena_run_t *run = &miscelm->run; + arena_run_split_large(arena, &miscelm->run, size, zero); return (run); } @@ -681,7 +668,7 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero) */ chunk = arena_chunk_alloc(arena); if (chunk != NULL) { - run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); + run = &arena_miscelm_get(chunk, map_bias)->run; arena_run_split_large(arena, run, size, zero); return (run); } @@ -704,11 +691,7 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (miscelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(miscelm); - size_t pageind = arena_miscelm_to_pageind(miscelm); - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - LG_PAGE)); + run = &miscelm->run; arena_run_split_small(arena, run, size, binind); return (run); } @@ -736,7 +719,7 @@ arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) */ chunk = arena_chunk_alloc(arena); if (chunk != NULL) { - run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); + run = &arena_miscelm_get(chunk, map_bias)->run; arena_run_split_small(arena, run, size, binind); return (run); } @@ -825,8 +808,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, size_t run_size = arena_mapbits_unallocated_size_get(chunk, pageind); size_t npages = run_size >> LG_PAGE; - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)(pageind << LG_PAGE)); + arena_run_t *run = &miscelm->run; assert(pageind + npages <= chunk_npages); assert(arena_mapbits_dirty_get(chunk, pageind) == @@ -919,11 +901,7 @@ arena_unstash_purged(arena_t *arena, arena_chunk_miscelms_t *miscelms) /* Deallocate runs. */ for (miscelm = ql_first(miscelms); miscelm != NULL; miscelm = ql_first(miscelms)) { - arena_chunk_t *chunk = - (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); - size_t pageind = arena_miscelm_to_pageind(miscelm); - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)(pageind << LG_PAGE)); + arena_run_t *run = &miscelm->run; ql_remove(miscelms, miscelm, dr_link); arena_run_dalloc(arena, run, false, true); } @@ -1042,10 +1020,12 @@ static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) { arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; size_t size, run_ind, run_pages, flag_dirty; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + miscelm = arena_run_to_miscelm(run); + run_ind = arena_miscelm_to_pageind(miscelm); assert(run_ind >= map_bias); assert(run_ind < chunk_npages); if (arena_mapbits_large_get(chunk, run_ind) != 0) { @@ -1086,8 +1066,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); } - arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, - flag_dirty); + arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, flag_dirty); /* Insert into runs_avail, now that coalescing is complete. */ assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == @@ -1121,7 +1100,8 @@ static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize) { - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); + size_t pageind = arena_miscelm_to_pageind(miscelm); size_t head_npages = (oldsize - newsize) >> LG_PAGE; size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); @@ -1153,9 +1133,12 @@ static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) { - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); + size_t pageind = arena_miscelm_to_pageind(miscelm); size_t head_npages = newsize >> LG_PAGE; size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); + arena_chunk_map_misc_t *tail_miscelm; + arena_run_t *tail_run; assert(oldsize > newsize); @@ -1178,26 +1161,17 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, flag_dirty); - arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), - dirty, false); + tail_miscelm = arena_miscelm_get(chunk, pageind + head_npages); + tail_run = &tail_miscelm->run; + arena_run_dalloc(arena, tail_run, dirty, false); } static arena_run_t * arena_bin_runs_first(arena_bin_t *bin) { arena_chunk_map_misc_t *miscelm = arena_run_tree_first(&bin->runs); - if (miscelm != NULL) { - arena_chunk_t *chunk; - size_t pageind; - arena_run_t *run; - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); - pageind = arena_miscelm_to_pageind(miscelm); - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - arena_mapbits_small_runind_get(chunk, pageind)) << - LG_PAGE)); - return (run); - } + if (miscelm != NULL) + return (&miscelm->run); return (NULL); } @@ -1205,9 +1179,7 @@ arena_bin_runs_first(arena_bin_t *bin) static void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) { - arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; - arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); assert(arena_run_tree_search(&bin->runs, miscelm) == NULL); @@ -1217,9 +1189,7 @@ arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) static void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) { - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; - arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); + arena_chunk_map_misc_t *miscelm = arena_run_to_miscelm(run); assert(arena_run_tree_search(&bin->runs, miscelm) != NULL); @@ -1260,14 +1230,11 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) malloc_mutex_lock(&arena->lock); run = arena_run_alloc_small(arena, bin_info->run_size, binind); if (run != NULL) { - bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + - (uintptr_t)bin_info->bitmap_offset); - /* Initialize run internals. */ run->bin = bin; run->nextind = 0; run->nfree = bin_info->nregs; - bitmap_init(bitmap, &bin_info->bitmap_info); + bitmap_init(run->bitmap, &bin_info->bitmap_info); } malloc_mutex_unlock(&arena->lock); /********************************/ @@ -1542,16 +1509,20 @@ void * arena_malloc_large(arena_t *arena, size_t size, bool zero) { void *ret; + arena_run_t *run; + arena_chunk_map_misc_t *miscelm; UNUSED bool idump; /* Large allocation. */ size = PAGE_CEILING(size); malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc_large(arena, size, zero); - if (ret == NULL) { + run = arena_run_alloc_large(arena, size, zero); + if (run == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); } + miscelm = arena_run_to_miscelm(run); + ret = arena_miscelm_to_rpages(miscelm); if (config_stats) { arena->stats.nmalloc_large++; arena->stats.nrequests_large++; @@ -1586,6 +1557,8 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) size_t alloc_size, leadsize, trailsize; arena_run_t *run; arena_chunk_t *chunk; + arena_chunk_map_misc_t *miscelm; + void *rpages; assert((size & PAGE_MASK) == 0); @@ -1599,21 +1572,31 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) return (NULL); } chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + miscelm = arena_run_to_miscelm(run); + rpages = arena_miscelm_to_rpages(miscelm); - leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - - (uintptr_t)run; + leadsize = ALIGNMENT_CEILING((uintptr_t)rpages, alignment) - + (uintptr_t)rpages; assert(alloc_size >= leadsize + size); trailsize = alloc_size - leadsize - size; - ret = (void *)((uintptr_t)run + leadsize); if (leadsize != 0) { - arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - - leadsize); + arena_chunk_map_misc_t *head_miscelm = miscelm; + arena_run_t *head_run = run; + + miscelm = arena_miscelm_get(chunk, + arena_miscelm_to_pageind(head_miscelm) + (leadsize >> + LG_PAGE)); + run = &miscelm->run; + + arena_run_trim_head(arena, chunk, head_run, alloc_size, + alloc_size - leadsize); } if (trailsize != 0) { - arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, + arena_run_trim_tail(arena, chunk, run, size + trailsize, size, false); } - arena_run_init_large(arena, (arena_run_t *)ret, size, zero); + arena_run_init_large(arena, run, size, zero); + ret = arena_miscelm_to_rpages(miscelm); if (config_stats) { arena->stats.nmalloc_large++; @@ -1687,10 +1670,12 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t binind; arena_bin_info_t *bin_info; size_t npages, run_ind, past; + arena_chunk_map_misc_t *miscelm; + void *rpages; assert(run != bin->runcur); - assert(arena_run_tree_search(&bin->runs, arena_miscelm_get(chunk, - ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) == NULL); + assert(arena_run_tree_search(&bin->runs, arena_run_to_miscelm(run)) == + NULL); binind = arena_bin_index(chunk->arena, run->bin); bin_info = &arena_bin_info[binind]; @@ -1698,8 +1683,10 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, malloc_mutex_unlock(&bin->lock); /******************************/ npages = bin_info->run_size >> LG_PAGE; - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); - past = (size_t)(PAGE_CEILING((uintptr_t)run + + miscelm = arena_run_to_miscelm(run); + run_ind = arena_miscelm_to_pageind(miscelm); + rpages = arena_miscelm_to_rpages(miscelm); + past = (size_t)(PAGE_CEILING((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * bin_info->reg_interval - bin_info->redzone_size) - (uintptr_t)chunk) >> LG_PAGE); @@ -1716,13 +1703,18 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, npages) { /* Trim clean pages. Convert to large run beforehand. */ assert(npages > 0); - arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); - arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); - arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), - ((past - run_ind) << LG_PAGE), false); + if (past > run_ind) { + arena_mapbits_large_set(chunk, run_ind, + bin_info->run_size, 0); + arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); + arena_run_trim_tail(arena, chunk, run, (npages << + LG_PAGE), ((past - run_ind) << LG_PAGE), false); + arena_run_dalloc(arena, run, true, false); + } else + arena_run_dalloc(arena, run, false, false); /* npages = past - run_ind; */ - } - arena_run_dalloc(arena, run, true, false); + } else + arena_run_dalloc(arena, run, true, false); malloc_mutex_unlock(&arena->lock); /****************************/ malloc_mutex_lock(&bin->lock); @@ -1755,15 +1747,15 @@ void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm) { - size_t pageind; + size_t pageind, rpages_ind; arena_run_t *run; arena_bin_t *bin; arena_bin_info_t *bin_info; size_t size, binind; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); + rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); + run = &arena_miscelm_get(chunk, rpages_ind)->run; bin = run->bin; binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)); @@ -1793,9 +1785,10 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, { arena_run_t *run; arena_bin_t *bin; + size_t rpages_ind; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); + rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); + run = &arena_miscelm_get(chunk, rpages_ind)->run; bin = run->bin; malloc_mutex_lock(&bin->lock); arena_dalloc_bin_locked(arena, chunk, ptr, bitselm); @@ -1838,9 +1831,11 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large = void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) { + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); + arena_run_t *run = &miscelm->run; if (config_fill || config_stats) { - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t usize = arena_mapbits_large_size_get(chunk, pageind); arena_dalloc_junk_large(ptr, usize); @@ -1852,7 +1847,7 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) } } - arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); + arena_run_dalloc(arena, run, true, false); } void @@ -1868,6 +1863,9 @@ static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t oldsize, size_t size) { + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); + arena_run_t *run = &miscelm->run; assert(size < oldsize); @@ -1876,8 +1874,7 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, * allocations. */ malloc_mutex_lock(&arena->lock); - arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, - true); + arena_run_trim_tail(arena, chunk, run, oldsize, size, true); if (config_stats) { arena->stats.ndalloc_large++; arena->stats.allocated_large -= oldsize; @@ -1919,8 +1916,9 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t flag_dirty; size_t splitsize = (oldsize + followsize <= size + extra) ? followsize : size + extra - oldsize; - arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + - ((pageind+npages) << LG_PAGE)), splitsize, zero); + arena_run_t *run = &arena_miscelm_get(chunk, + pageind+npages)->run; + arena_run_split_large(arena, run, splitsize, zero); size = oldsize + splitsize; npages = size >> LG_PAGE; @@ -2249,26 +2247,18 @@ arena_new(arena_t *arena, unsigned ind) /* * Calculate bin_info->run_size such that it meets the following constraints: * - * *) bin_info->run_size >= min_run_size * *) bin_info->run_size <= arena_maxclass - * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). * *) bin_info->nregs <= RUN_MAXREGS * - * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also - * calculated here, since these settings are all interdependent. + * bin_info->nregs and bin_info->reg0_offset are also calculated here, since + * these settings are all interdependent. */ -static size_t -bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) +static void +bin_info_run_size_calc(arena_bin_info_t *bin_info) { size_t pad_size; - size_t try_run_size, good_run_size; - uint32_t try_nregs, good_nregs; - uint32_t try_hdr_size, good_hdr_size; - uint32_t try_bitmap_offset, good_bitmap_offset; - uint32_t try_redzone0_offset, good_redzone0_offset; - - assert(min_run_size >= PAGE); - assert(min_run_size <= arena_maxclass); + size_t try_run_size, perfect_run_size, actual_run_size; + uint32_t try_nregs, perfect_nregs, actual_nregs; /* * Determine redzone size based on minimum alignment and minimum @@ -2295,96 +2285,66 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) (bin_info->redzone_size << 1); /* - * Calculate known-valid settings before entering the run_size - * expansion loop, so that the first part of the loop always copies - * valid settings. - * - * The do..while loop iteratively reduces the number of regions until - * the run header and the regions no longer overlap. A closed formula - * would be quite messy, since there is an interdependency between the - * header's mask length and the number of regions. + * Compute run size under ideal conditions (no redzones, no limit on run + * size). */ - try_run_size = min_run_size; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / - bin_info->reg_interval) - + 1; /* Counter-act try_nregs-- in loop. */ - if (try_nregs > RUN_MAXREGS) { - try_nregs = RUN_MAXREGS - + 1; /* Counter-act try_nregs-- in loop. */ - } + try_run_size = PAGE; + try_nregs = try_run_size / bin_info->reg_size; do { - try_nregs--; - try_hdr_size = sizeof(arena_run_t); - /* Pad to a long boundary. */ - try_hdr_size = LONG_CEILING(try_hdr_size); - try_bitmap_offset = try_hdr_size; - /* Add space for bitmap. */ - try_hdr_size += bitmap_size(try_nregs); - try_redzone0_offset = try_run_size - (try_nregs * - bin_info->reg_interval) - pad_size; - } while (try_hdr_size > try_redzone0_offset); + perfect_run_size = try_run_size; + perfect_nregs = try_nregs; - /* run_size expansion loop. */ - do { - /* - * Copy valid settings before trying more aggressive settings. - */ - good_run_size = try_run_size; - good_nregs = try_nregs; - good_hdr_size = try_hdr_size; - good_bitmap_offset = try_bitmap_offset; - good_redzone0_offset = try_redzone0_offset; - - /* Try more aggressive settings. */ try_run_size += PAGE; - try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / - bin_info->reg_interval) - + 1; /* Counter-act try_nregs-- in loop. */ - if (try_nregs > RUN_MAXREGS) { - try_nregs = RUN_MAXREGS - + 1; /* Counter-act try_nregs-- in loop. */ - } - do { - try_nregs--; - try_hdr_size = sizeof(arena_run_t); - /* Pad to a long boundary. */ - try_hdr_size = LONG_CEILING(try_hdr_size); - try_bitmap_offset = try_hdr_size; - /* Add space for bitmap. */ - try_hdr_size += bitmap_size(try_nregs); - try_redzone0_offset = try_run_size - (try_nregs * - bin_info->reg_interval) - pad_size; - } while (try_hdr_size > try_redzone0_offset); - } while (try_run_size <= arena_maxclass - && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > - RUN_MAX_OVRHD_RELAX - && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size - && try_nregs < RUN_MAXREGS); + try_nregs = try_run_size / bin_info->reg_size; + } while (perfect_run_size != perfect_nregs * bin_info->reg_size); + assert(perfect_nregs <= RUN_MAXREGS); - assert(good_hdr_size <= good_redzone0_offset); + actual_run_size = perfect_run_size; + actual_nregs = (actual_run_size - pad_size) / bin_info->reg_interval; + + /* + * Redzones can require enough padding that not even a single region can + * fit within the number of pages that would normally be dedicated to a + * run for this size class. Increase the run size until at least one + * region fits. + */ + while (actual_nregs == 0) { + assert(config_fill && unlikely(opt_redzone)); + + actual_run_size += PAGE; + actual_nregs = (actual_run_size - pad_size) / + bin_info->reg_interval; + } + + /* + * Make sure that the run will fit within an arena chunk. + */ + while (actual_run_size > arena_maxclass) { + actual_run_size -= PAGE; + actual_nregs = (actual_run_size - pad_size) / + bin_info->reg_interval; + } + assert(actual_nregs > 0); /* Copy final settings. */ - bin_info->run_size = good_run_size; - bin_info->nregs = good_nregs; - bin_info->bitmap_offset = good_bitmap_offset; - bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; + bin_info->run_size = actual_run_size; + bin_info->nregs = actual_nregs; + bin_info->reg0_offset = actual_run_size - (actual_nregs * + bin_info->reg_interval) - pad_size + bin_info->redzone_size; assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs * bin_info->reg_interval) + pad_size == bin_info->run_size); - - return (good_run_size); } static void bin_info_init(void) { arena_bin_info_t *bin_info; - size_t prev_run_size = PAGE; #define BIN_INFO_INIT_bin_yes(index, size) \ bin_info = &arena_bin_info[index]; \ bin_info->reg_size = size; \ - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ + bin_info_run_size_calc(bin_info); \ bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); #define BIN_INFO_INIT_bin_no(index, size) #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ @@ -2418,8 +2378,7 @@ arena_boot(void) header_size = offsetof(arena_chunk_t, map_bits) + ((sizeof(arena_chunk_map_bits_t) + sizeof(arena_chunk_map_misc_t)) * (chunk_npages-map_bias)); - map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) - != 0); + map_bias = (header_size + PAGE_MASK) >> LG_PAGE; } assert(map_bias > 0); From e3a16fce5eb0c62a49e751f156d040c9f77fbc23 Mon Sep 17 00:00:00 2001 From: Dave Rigby Date: Wed, 24 Sep 2014 14:19:28 +0100 Subject: [PATCH 119/352] Mark malloc_conf as a weak symbol This fixes issue #113 - je_malloc_conf is not respected on OS X --- src/jemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 4d3b22e5..3012f558 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -7,7 +7,7 @@ malloc_tsd_data(, arenas, arena_t *, NULL) /* Runtime configuration options. */ -const char *je_malloc_conf; +const char *je_malloc_conf JEMALLOC_ATTR(weak); bool opt_abort = #ifdef JEMALLOC_DEBUG true From 4dcf04bfc03b9e9eb50015a8fc8735de28c23090 Mon Sep 17 00:00:00 2001 From: Eric Wong Date: Sun, 31 Aug 2014 03:57:06 +0000 Subject: [PATCH 120/352] correctly detect adaptive mutexes in pthreads PTHREAD_MUTEX_ADAPTIVE_NP is an enum on glibc and not a macro, we must test for their existence by attempting compilation. --- configure.ac | 12 ++++++++++++ .../jemalloc/internal/jemalloc_internal_defs.h.in | 3 +++ include/jemalloc/internal/mutex.h | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index ab4bcc39..1ee2ed8e 100644 --- a/configure.ac +++ b/configure.ac @@ -1399,6 +1399,18 @@ if test "x${je_cv_glibc_memalign_hook}" = "xyes" ; then AC_DEFINE([JEMALLOC_GLIBC_MEMALIGN_HOOK], [ ]) fi +JE_COMPILABLE([pthreads adaptive mutexes], [ +#include +], [ + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP); + pthread_mutexattr_destroy(&attr); +], [je_cv_pthread_mutex_adaptive_np]) +if test "x${je_cv_pthread_mutex_adaptive_np}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP], [ ]) +fi + dnl ============================================================================ dnl Check for typedefs, structures, and compiler characteristics. AC_HEADER_STDBOOL diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 955582ee..fd85e5cf 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -215,4 +215,7 @@ /* glibc memalign hook */ #undef JEMALLOC_GLIBC_MEMALIGN_HOOK +/* adaptive mutex support in pthreads */ +#undef JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/include/jemalloc/internal/mutex.h b/include/jemalloc/internal/mutex.h index de44e143..8a03d825 100644 --- a/include/jemalloc/internal/mutex.h +++ b/include/jemalloc/internal/mutex.h @@ -10,7 +10,7 @@ typedef struct malloc_mutex_s malloc_mutex_t; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) # define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL} #else -# if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \ +# if (defined(JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP) && \ defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP)) # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP # define MALLOC_MUTEX_INITIALIZER {PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP} From f8034540a16a6f4fc7948e4783747ca1e9055823 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Tue, 30 Sep 2014 10:33:46 -0400 Subject: [PATCH 121/352] Implement in-place huge allocation shrinking. Trivial example: #include int main(void) { void *ptr = malloc(1024 * 1024 * 8); if (!ptr) return 1; ptr = realloc(ptr, 1024 * 1024 * 4); if (!ptr) return 1; } Before: mmap(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fcfff000000 mmap(NULL, 4194304, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fcffec00000 madvise(0x7fcfff000000, 8388608, MADV_DONTNEED) = 0 After: mmap(NULL, 8388608, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f1934800000 madvise(0x7f1934c00000, 4194304, MADV_DONTNEED) = 0 Closes #134 --- src/huge.c | 89 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 27 deletions(-) diff --git a/src/huge.c b/src/huge.c index 2e30ccfd..40d1362d 100644 --- a/src/huge.c +++ b/src/huge.c @@ -72,21 +72,79 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, return (ret); } +#ifdef JEMALLOC_JET +#undef huge_dalloc_junk +#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) +#endif +static void +huge_dalloc_junk(void *ptr, size_t usize) +{ + + if (config_fill && have_dss && unlikely(opt_junk)) { + /* + * Only bother junk filling if the chunk isn't about to be + * unmapped. + */ + if (config_munmap == false || (have_dss && chunk_in_dss(ptr))) + memset(ptr, 0x5a, usize); + } +} +#ifdef JEMALLOC_JET +#undef huge_dalloc_junk +#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) +huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); +#endif + bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) { + /* Both allocations must be huge to avoid a move. */ + if (oldsize <= arena_maxclass) + return (true); + + assert(CHUNK_CEILING(oldsize) == oldsize); + /* * Avoid moving the allocation if the size class can be left the same. */ - if (oldsize > arena_maxclass - && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) + if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { - assert(CHUNK_CEILING(oldsize) == oldsize); return (false); } - /* Reallocation would require a move. */ + /* Overflow. */ + if (CHUNK_CEILING(size) == 0) + return (true); + + /* Shrink the allocation in-place. */ + if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(size)) { + extent_node_t *node, key; + void *excess_addr; + size_t excess_size; + + malloc_mutex_lock(&huge_mtx); + + key.addr = ptr; + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + + /* Update the size of the huge allocation. */ + node->size = CHUNK_CEILING(size); + + malloc_mutex_unlock(&huge_mtx); + + excess_addr = node->addr + CHUNK_CEILING(size); + excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(size); + + /* Zap the excess chunks. */ + huge_dalloc_junk(excess_addr, excess_size); + arena_chunk_dalloc_huge(node->arena, excess_addr, excess_size); + + return (false); + } + return (true); } @@ -134,29 +192,6 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, return (ret); } -#ifdef JEMALLOC_JET -#undef huge_dalloc_junk -#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) -#endif -static void -huge_dalloc_junk(void *ptr, size_t usize) -{ - - if (config_fill && have_dss && unlikely(opt_junk)) { - /* - * Only bother junk filling if the chunk isn't about to be - * unmapped. - */ - if (config_munmap == false || (have_dss && chunk_in_dss(ptr))) - memset(ptr, 0x5a, usize); - } -} -#ifdef JEMALLOC_JET -#undef huge_dalloc_junk -#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) -huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); -#endif - void huge_dalloc(void *ptr) { From cc9e626ea97eb294f337c674685b8b5c9d5524b7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 1 Oct 2014 17:51:52 -0700 Subject: [PATCH 122/352] Refactor permuted backtrace test allocation. Refactor permuted backtrace test allocation that was originally used only by the prof_accum test, so that it can be used by other heap profiling test binaries. --- Makefile.in | 20 ++++++---------- test/include/test/btalloc.h | 31 ++++++++++++++++++++++++ test/include/test/jemalloc_test.h.in | 1 + test/src/btalloc.c | 8 +++++++ test/src/btalloc_0.c | 3 +++ test/src/btalloc_1.c | 3 +++ test/unit/prof_accum.c | 9 +++++-- test/unit/prof_accum.h | 35 ---------------------------- test/unit/prof_accum_a.c | 3 --- test/unit/prof_accum_b.c | 3 --- 10 files changed, 60 insertions(+), 56 deletions(-) create mode 100644 test/include/test/btalloc.h create mode 100644 test/src/btalloc.c create mode 100644 test/src/btalloc_0.c create mode 100644 test/src/btalloc_1.c delete mode 100644 test/unit/prof_accum.h delete mode 100644 test/unit/prof_accum_a.c delete mode 100644 test/unit/prof_accum_b.c diff --git a/Makefile.in b/Makefile.in index 41328b95..5267bea4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -107,9 +107,11 @@ DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3) DOCS := $(DOCS_HTML) $(DOCS_MAN3) -C_TESTLIB_SRCS := $(srcroot)test/src/math.c $(srcroot)test/src/mtx.c \ - $(srcroot)test/src/SFMT.c $(srcroot)test/src/test.c \ - $(srcroot)test/src/thd.c $(srcroot)test/src/timer.c +C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \ + $(srcroot)test/src/btalloc_1.c $(srcroot)test/src/math.c \ + $(srcroot)test/src/mtx.c $(srcroot)test/src/SFMT.c \ + $(srcroot)test/src/test.c $(srcroot)test/src/thd.c \ + $(srcroot)test/src/timer.c C_UTIL_INTEGRATION_SRCS := $(srcroot)src/util.c TESTS_UNIT := $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/bitmap.c \ @@ -123,6 +125,7 @@ TESTS_UNIT := $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/prof_accum.c \ $(srcroot)test/unit/prof_gdump.c \ $(srcroot)test/unit/prof_idump.c \ + $(srcroot)test/unit/prof_reset.c \ $(srcroot)test/unit/ql.c \ $(srcroot)test/unit/qr.c \ $(srcroot)test/unit/quarantine.c \ @@ -133,8 +136,6 @@ TESTS_UNIT := $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/tsd.c \ $(srcroot)test/unit/util.c \ $(srcroot)test/unit/zero.c -TESTS_UNIT_AUX := $(srcroot)test/unit/prof_accum_a.c \ - $(srcroot)test/unit/prof_accum_b.c TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/allocated.c \ $(srcroot)test/integration/sdallocx.c \ @@ -159,10 +160,9 @@ C_TESTLIB_STRESS_OBJS := $(C_TESTLIB_SRCS:$(srcroot)%.c=$(objroot)%.stress.$(O)) C_TESTLIB_OBJS := $(C_TESTLIB_UNIT_OBJS) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(C_TESTLIB_STRESS_OBJS) TESTS_UNIT_OBJS := $(TESTS_UNIT:$(srcroot)%.c=$(objroot)%.$(O)) -TESTS_UNIT_AUX_OBJS := $(TESTS_UNIT_AUX:$(srcroot)%.c=$(objroot)%.$(O)) TESTS_INTEGRATION_OBJS := $(TESTS_INTEGRATION:$(srcroot)%.c=$(objroot)%.$(O)) TESTS_STRESS_OBJS := $(TESTS_STRESS:$(srcroot)%.c=$(objroot)%.$(O)) -TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_UNIT_AUX_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS) +TESTS_OBJS := $(TESTS_UNIT_OBJS) $(TESTS_INTEGRATION_OBJS) $(TESTS_STRESS_OBJS) .PHONY: all dist build_doc_html build_doc_man build_doc .PHONY: install_bin install_include install_lib @@ -211,12 +211,6 @@ $(C_TESTLIB_STRESS_OBJS): $(objroot)test/src/%.stress.$(O): $(srcroot)test/src/% $(C_TESTLIB_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST -DJEMALLOC_STRESS_TESTLIB $(C_TESTLIB_OBJS): CPPFLAGS += -I$(srcroot)test/include -I$(objroot)test/include $(TESTS_UNIT_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST -$(TESTS_UNIT_AUX_OBJS): CPPFLAGS += -DJEMALLOC_UNIT_TEST -define make-unit-link-dep -$(1): TESTS_UNIT_LINK_OBJS += $(2) -$(1): $(2) -endef -$(foreach test, $(TESTS_UNIT:$(srcroot)test/unit/%.c=$(objroot)test/unit/%$(EXE)), $(eval $(call make-unit-link-dep,$(test),$(filter $(test:%$(EXE)=%_a.$(O)) $(test:%$(EXE)=%_b.$(O)),$(TESTS_UNIT_AUX_OBJS))))) $(TESTS_INTEGRATION_OBJS): CPPFLAGS += -DJEMALLOC_INTEGRATION_TEST $(TESTS_STRESS_OBJS): CPPFLAGS += -DJEMALLOC_STRESS_TEST $(TESTS_OBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c diff --git a/test/include/test/btalloc.h b/test/include/test/btalloc.h new file mode 100644 index 00000000..c3f9d4df --- /dev/null +++ b/test/include/test/btalloc.h @@ -0,0 +1,31 @@ +/* btalloc() provides a mechanism for allocating via permuted backtraces. */ +void *btalloc(size_t size, unsigned bits); + +#define btalloc_n_proto(n) \ +void *btalloc_##n(size_t size, unsigned bits); +btalloc_n_proto(0) +btalloc_n_proto(1) + +#define btalloc_n_gen(n) \ +void * \ +btalloc_##n(size_t size, unsigned bits) \ +{ \ + void *p; \ + \ + if (bits == 0) \ + p = mallocx(size, 0); \ + else { \ + switch (bits & 0x1U) { \ + case 0: \ + p = (btalloc_0(size, bits >> 1)); \ + break; \ + case 1: \ + p = (btalloc_1(size, bits >> 1)); \ + break; \ + default: not_reached(); \ + } \ + } \ + /* Intentionally sabotage tail call optimization. */ \ + assert_ptr_not_null(p, "Unexpected mallocx() failure"); \ + return (p); \ +} diff --git a/test/include/test/jemalloc_test.h.in b/test/include/test/jemalloc_test.h.in index a93c4f67..6018e58a 100644 --- a/test/include/test/jemalloc_test.h.in +++ b/test/include/test/jemalloc_test.h.in @@ -133,6 +133,7 @@ /* * Common test utilities. */ +#include "test/btalloc.h" #include "test/math.h" #include "test/mtx.h" #include "test/mq.h" diff --git a/test/src/btalloc.c b/test/src/btalloc.c new file mode 100644 index 00000000..9a253d97 --- /dev/null +++ b/test/src/btalloc.c @@ -0,0 +1,8 @@ +#include "test/jemalloc_test.h" + +void * +btalloc(size_t size, unsigned bits) +{ + + return (btalloc_0(size, bits)); +} diff --git a/test/src/btalloc_0.c b/test/src/btalloc_0.c new file mode 100644 index 00000000..77d8904e --- /dev/null +++ b/test/src/btalloc_0.c @@ -0,0 +1,3 @@ +#include "test/jemalloc_test.h" + +btalloc_n_gen(0) diff --git a/test/src/btalloc_1.c b/test/src/btalloc_1.c new file mode 100644 index 00000000..4c126c30 --- /dev/null +++ b/test/src/btalloc_1.c @@ -0,0 +1,3 @@ +#include "test/jemalloc_test.h" + +btalloc_n_gen(1) diff --git a/test/unit/prof_accum.c b/test/unit/prof_accum.c index 050a8a7e..fd229e0f 100644 --- a/test/unit/prof_accum.c +++ b/test/unit/prof_accum.c @@ -1,4 +1,9 @@ -#include "prof_accum.h" +#include "test/jemalloc_test.h" + +#define NTHREADS 4 +#define NALLOCS_PER_THREAD 50 +#define DUMP_INTERVAL 1 +#define BT_COUNT_CHECK_INTERVAL 5 #ifdef JEMALLOC_PROF const char *malloc_conf = @@ -20,7 +25,7 @@ static void * alloc_from_permuted_backtrace(unsigned thd_ind, unsigned iteration) { - return (alloc_0(thd_ind*NALLOCS_PER_THREAD + iteration)); + return (btalloc(1, thd_ind*NALLOCS_PER_THREAD + iteration)); } static void * diff --git a/test/unit/prof_accum.h b/test/unit/prof_accum.h deleted file mode 100644 index 109d86b5..00000000 --- a/test/unit/prof_accum.h +++ /dev/null @@ -1,35 +0,0 @@ -#include "test/jemalloc_test.h" - -#define NTHREADS 4 -#define NALLOCS_PER_THREAD 50 -#define DUMP_INTERVAL 1 -#define BT_COUNT_CHECK_INTERVAL 5 - -#define alloc_n_proto(n) \ -void *alloc_##n(unsigned bits); -alloc_n_proto(0) -alloc_n_proto(1) - -#define alloc_n_gen(n) \ -void * \ -alloc_##n(unsigned bits) \ -{ \ - void *p; \ - \ - if (bits == 0) \ - p = mallocx(1, 0); \ - else { \ - switch (bits & 0x1U) { \ - case 0: \ - p = (alloc_0(bits >> 1)); \ - break; \ - case 1: \ - p = (alloc_1(bits >> 1)); \ - break; \ - default: not_reached(); \ - } \ - } \ - /* Intentionally sabotage tail call optimization. */ \ - assert_ptr_not_null(p, "Unexpected mallocx() failure"); \ - return (p); \ -} diff --git a/test/unit/prof_accum_a.c b/test/unit/prof_accum_a.c deleted file mode 100644 index 42ad521d..00000000 --- a/test/unit/prof_accum_a.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "prof_accum.h" - -alloc_n_gen(0) diff --git a/test/unit/prof_accum_b.c b/test/unit/prof_accum_b.c deleted file mode 100644 index 60d9dab6..00000000 --- a/test/unit/prof_accum_b.c +++ /dev/null @@ -1,3 +0,0 @@ -#include "prof_accum.h" - -alloc_n_gen(1) From 20c31deaae38ed9aa4fe169ed65e0c45cd542955 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 2 Oct 2014 23:01:10 -0700 Subject: [PATCH 123/352] Test prof.reset mallctl and fix numerous discovered bugs. --- doc/jemalloc.xml.in | 5 +- include/jemalloc/internal/private_symbols.txt | 1 + include/jemalloc/internal/prof.h | 24 +- src/prof.c | 211 +++++++++++----- test/unit/prof_reset.c | 238 ++++++++++++++++++ 5 files changed, 404 insertions(+), 75 deletions(-) create mode 100644 test/unit/prof_reset.c diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index e5c229fe..b586e690 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1547,7 +1547,8 @@ malloc_conf = "xmalloc:true";]]> Reset all memory profile statistics, and optionally update the sample rate (see opt.lg_prof_sample). + linkend="opt.lg_prof_sample">opt.lg_prof_sample + and prof.lg_sample). @@ -1558,7 +1559,7 @@ malloc_conf = "xmalloc:true";]]> r- [] - Get the sample rate (see Get the current sample rate (see opt.lg_prof_sample). diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 5ac82f59..33f8ce01 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -292,6 +292,7 @@ prof_boot0 prof_boot1 prof_boot2 prof_bt_count +prof_dump_header prof_dump_open prof_free prof_free_sampled_object diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 3872c7ae..91c871de 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -151,22 +151,23 @@ struct prof_gctx_s { }; typedef rb_tree(prof_gctx_t) prof_gctx_tree_t; -typedef enum { - prof_tdata_state_attached, /* Active thread attached, data valid. */ - prof_tdata_state_detached, /* Defunct thread, data remain valid. */ - prof_tdata_state_expired /* Predates reset, omit data from dump. */ -} prof_tdata_state_t; - struct prof_tdata_s { malloc_mutex_t *lock; /* Monotonically increasing unique thread identifier. */ uint64_t thr_uid; + /* + * Monotonically increasing discriminator among tdata structures + * associated with the same thr_uid. + */ + uint64_t thr_discrim; + /* Included in heap profile dumps if non-NULL. */ char *thread_name; - prof_tdata_state_t state; + bool attached; + bool expired; rb_node(prof_tdata_t) tdata_link; @@ -257,9 +258,13 @@ void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt); prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); #ifdef JEMALLOC_JET +size_t prof_tdata_count(void); size_t prof_bt_count(void); +const prof_cnt_t *prof_cnt_all(void); typedef int (prof_dump_open_t)(bool, const char *); extern prof_dump_open_t *prof_dump_open; +typedef bool (prof_dump_header_t)(bool, const prof_cnt_t *); +extern prof_dump_header_t *prof_dump_header; #endif void prof_idump(void); bool prof_mdump(const char *filename); @@ -312,12 +317,11 @@ prof_tdata_get(tsd_t *tsd, bool create) if (unlikely(tdata == NULL)) { tdata = prof_tdata_init(tsd); tsd_prof_tdata_set(tsd, tdata); - } else if (unlikely(tdata->state == prof_tdata_state_expired)) { + } else if (unlikely(tdata->expired)) { tdata = prof_tdata_reinit(tsd, tdata); tsd_prof_tdata_set(tsd, tdata); } - assert(tdata == NULL || - tdata->state == prof_tdata_state_attached); + assert(tdata == NULL || tdata->attached); } return (tdata); diff --git a/src/prof.c b/src/prof.c index 9f10b533..0a96d85f 100644 --- a/src/prof.c +++ b/src/prof.c @@ -137,10 +137,18 @@ rb_gen(static UNUSED, gctx_tree_, prof_gctx_tree_t, prof_gctx_t, dump_link, JEMALLOC_INLINE_C int prof_tdata_comp(const prof_tdata_t *a, const prof_tdata_t *b) { + int ret; uint64_t a_uid = a->thr_uid; uint64_t b_uid = b->thr_uid; - return ((a_uid > b_uid) - (a_uid < b_uid)); + ret = ((a_uid > b_uid) - (a_uid < b_uid)); + if (ret == 0) { + uint64_t a_discrim = a->thr_discrim; + uint64_t b_discrim = b->thr_discrim; + + ret = ((a_discrim > b_discrim) - (a_discrim < b_discrim)); + } + return (ret); } rb_gen(static UNUSED, tdata_tree_, prof_tdata_tree_t, prof_tdata_t, tdata_link, @@ -504,7 +512,7 @@ prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) gctx->lock = prof_gctx_mutex_choose(); /* * Set nlimbo to 1, in order to avoid a race condition with - * prof_tctx_destroy()/prof_gctx_maybe_destroy(). + * prof_tctx_destroy()/prof_gctx_try_destroy(). */ gctx->nlimbo = 1; tctx_tree_new(&gctx->tctxs); @@ -516,7 +524,7 @@ prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) } static void -prof_gctx_maybe_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata) +prof_gctx_try_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata) { cassert(config_prof); @@ -530,6 +538,7 @@ prof_gctx_maybe_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata) */ prof_enter(tdata); malloc_mutex_lock(gctx->lock); + assert(gctx->nlimbo != 0); if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { /* Remove gctx from bt2gctx. */ if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) @@ -605,10 +614,10 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) * * 1) Sample an allocation associated with gctx. * 2) Deallocate the sampled object. - * 3) Successfully prof_gctx_maybe_destroy(gctx). + * 3) Successfully prof_gctx_try_destroy(gctx). * * The result would be that gctx no longer exists by the time - * this thread accesses it in prof_gctx_maybe_destroy(). + * this thread accesses it in prof_gctx_try_destroy(). */ gctx->nlimbo++; destroy_gctx = true; @@ -616,7 +625,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) destroy_gctx = false; malloc_mutex_unlock(gctx->lock); if (destroy_gctx) - prof_gctx_maybe_destroy(tsd, gctx, tdata); + prof_gctx_try_destroy(tsd, gctx, tdata); if (destroy_tdata) prof_tdata_destroy(tsd, tdata); @@ -657,7 +666,7 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, } else { /* * Increment nlimbo, in order to avoid a race condition with - * prof_tctx_destroy()/prof_gctx_maybe_destroy(). + * prof_tctx_destroy()/prof_gctx_try_destroy(). */ malloc_mutex_lock(gctx.p->lock); gctx.p->nlimbo++; @@ -710,7 +719,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) ret.v = imalloc(tsd, sizeof(prof_tctx_t)); if (ret.p == NULL) { if (new_gctx) - prof_gctx_maybe_destroy(tsd, gctx, tdata); + prof_gctx_try_destroy(tsd, gctx, tdata); return (NULL); } ret.p->tdata = tdata; @@ -723,7 +732,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) malloc_mutex_unlock(tdata->lock); if (error) { if (new_gctx) - prof_gctx_maybe_destroy(tsd, gctx, tdata); + prof_gctx_try_destroy(tsd, gctx, tdata); idalloc(tsd, ret.v); return (NULL); } @@ -792,6 +801,31 @@ prof_sample_threshold_update(prof_tdata_t *tdata) #endif } +#ifdef JEMALLOC_JET +static prof_tdata_t * +prof_tdata_count_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) +{ + size_t *tdata_count = (size_t *)arg; + + (*tdata_count)++; + + return (NULL); +} + +size_t +prof_tdata_count(void) +{ + size_t tdata_count = 0; + + malloc_mutex_lock(&tdatas_mtx); + tdata_tree_iter(&tdatas, NULL, prof_tdata_count_iter, + (void *)&tdata_count); + malloc_mutex_unlock(&tdatas_mtx); + + return (tdata_count); +} +#endif + #ifdef JEMALLOC_JET size_t prof_bt_count(void) @@ -998,7 +1032,6 @@ static prof_tctx_t * prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) { prof_tctx_t *ret; - tsd_t *tsd = (tsd_t *)arg; switch (tctx->state) { case prof_tctx_state_nominal: @@ -1008,9 +1041,7 @@ prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) tctx->state = prof_tctx_state_nominal; break; case prof_tctx_state_purgatory: - ret = tctx_tree_next(tctxs, tctx); - tctx_tree_remove(tctxs, tctx); - idalloc(tsd, tctx); + ret = tctx; goto label_return; default: not_reached(); @@ -1056,27 +1087,47 @@ prof_gctx_merge_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) return (NULL); } -static prof_gctx_t * -prof_gctx_finish_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) +static void +prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) { - tsd_t *tsd = (tsd_t *)arg; prof_tdata_t *tdata = prof_tdata_get(tsd, false); - prof_tctx_t *next; - bool destroy_gctx; + prof_gctx_t *gctx; - malloc_mutex_lock(gctx->lock); - next = NULL; - do { - next = tctx_tree_iter(&gctx->tctxs, next, prof_tctx_finish_iter, - tsd); - } while (next != NULL); - gctx->nlimbo--; - destroy_gctx = prof_gctx_should_destroy(gctx); - malloc_mutex_unlock(gctx->lock); - if (destroy_gctx) - prof_gctx_maybe_destroy(tsd, gctx, tdata); + /* + * Standard tree iteration won't work here, because as soon as we + * decrement gctx->nlimbo and unlock gctx, another thread can + * concurrently destroy it, which will corrupt the tree. Therefore, + * tear down the tree one node at a time during iteration. + */ + while ((gctx = gctx_tree_first(gctxs)) != NULL) { + gctx_tree_remove(gctxs, gctx); + malloc_mutex_lock(gctx->lock); + { + prof_tctx_t *next; - return (NULL); + next = NULL; + do { + prof_tctx_t *to_destroy = + tctx_tree_iter(&gctx->tctxs, next, + prof_tctx_finish_iter, NULL); + if (to_destroy != NULL) { + next = tctx_tree_next(&gctx->tctxs, + to_destroy); + tctx_tree_remove(&gctx->tctxs, + to_destroy); + idalloc(tsd, to_destroy); + } else + next = NULL; + } while (next != NULL); + } + gctx->nlimbo--; + if (prof_gctx_should_destroy(gctx)) { + gctx->nlimbo++; + malloc_mutex_unlock(gctx->lock); + prof_gctx_try_destroy(tsd, gctx, tdata); + } else + malloc_mutex_unlock(gctx->lock); + } } static prof_tdata_t * @@ -1085,7 +1136,7 @@ prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) prof_cnt_t *cnt_all = (prof_cnt_t *)arg; malloc_mutex_lock(tdata->lock); - if (tdata->state != prof_tdata_state_expired) { + if (!tdata->expired) { size_t tabind; union { prof_tctx_t *p; @@ -1130,6 +1181,10 @@ prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) return (NULL); } +#ifdef JEMALLOC_JET +#undef prof_dump_header +#define prof_dump_header JEMALLOC_N(prof_dump_header_impl) +#endif static bool prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) { @@ -1148,6 +1203,11 @@ prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) malloc_mutex_unlock(&tdatas_mtx); return (ret); } +#ifdef JEMALLOC_JET +#undef prof_dump_header +#define prof_dump_header JEMALLOC_N(prof_dump_header) +prof_dump_header_t *prof_dump_header = JEMALLOC_N(prof_dump_header_impl); +#endif /* gctx->lock is held. */ static bool @@ -1277,7 +1337,7 @@ prof_gctx_dump_iter(prof_gctx_tree_t *gctxs, prof_gctx_t *gctx, void *arg) malloc_mutex_lock(gctx->lock); if (prof_dump_gctx(propagate_err, gctx, &gctx->bt, gctxs)) { - ret = gctx_tree_next(gctxs, gctx); + ret = gctx; goto label_return; } @@ -1302,7 +1362,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) cassert(config_prof); - tdata = prof_tdata_get(tsd, false); + tdata = prof_tdata_get(tsd, true); if (tdata == NULL) return (true); @@ -1352,7 +1412,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) if (prof_dump_close(propagate_err)) goto label_open_close_error; - gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd); + prof_gctx_finish(tsd, &gctxs); malloc_mutex_unlock(&prof_dump_mtx); if (leakcheck) @@ -1362,7 +1422,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) label_write_error: prof_dump_close(propagate_err); label_open_close_error: - gctx_tree_iter(&gctxs, NULL, prof_gctx_finish_iter, tsd); + prof_gctx_finish(tsd, &gctxs); malloc_mutex_unlock(&prof_dump_mtx); return (true); } @@ -1533,7 +1593,7 @@ prof_thr_uid_alloc(void) } static prof_tdata_t * -prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid) +prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim) { prof_tdata_t *tdata; @@ -1546,8 +1606,10 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid) tdata->lock = prof_tdata_mutex_choose(thr_uid); tdata->thr_uid = thr_uid; + tdata->thr_discrim = thr_discrim; tdata->thread_name = NULL; - tdata->state = prof_tdata_state_attached; + tdata->attached = true; + tdata->expired = false; if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) { @@ -1576,14 +1638,7 @@ prof_tdata_t * prof_tdata_init(tsd_t *tsd) { - return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc())); -} - -prof_tdata_t * -prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) -{ - - return (prof_tdata_init_impl(tsd, tdata->thr_uid)); + return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0)); } /* tdata->lock must be held. */ @@ -1591,22 +1646,21 @@ static bool prof_tdata_should_destroy(prof_tdata_t *tdata) { - if (tdata->state == prof_tdata_state_attached) + if (tdata->attached) return (false); if (ckh_count(&tdata->bt2tctx) != 0) return (false); return (true); } +/* tdatas_mtx must be held. */ static void -prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata) +prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata) { assert(prof_tdata_should_destroy(tdata)); - malloc_mutex_lock(&tdatas_mtx); tdata_tree_remove(&tdatas, tdata); - malloc_mutex_unlock(&tdatas_mtx); if (tdata->thread_name != NULL) idalloc(tsd, tdata->thread_name); @@ -1615,14 +1669,22 @@ prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata) } static void -prof_tdata_state_transition(tsd_t *tsd, prof_tdata_t *tdata, - prof_tdata_state_t state) +prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata) +{ + + malloc_mutex_lock(&tdatas_mtx); + prof_tdata_destroy_locked(tsd, tdata); + malloc_mutex_unlock(&tdatas_mtx); +} + +static void +prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) { bool destroy_tdata; malloc_mutex_lock(tdata->lock); - if (tdata->state != state) { - tdata->state = state; + if (tdata->attached) { + tdata->attached = false; destroy_tdata = prof_tdata_should_destroy(tdata); } else destroy_tdata = false; @@ -1631,32 +1693,44 @@ prof_tdata_state_transition(tsd_t *tsd, prof_tdata_t *tdata, prof_tdata_destroy(tsd, tdata); } -static void -prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) +prof_tdata_t * +prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) { + uint64_t thr_uid = tdata->thr_uid; + uint64_t thr_discrim = tdata->thr_discrim + 1; - prof_tdata_state_transition(tsd, tdata, prof_tdata_state_detached); + prof_tdata_detach(tsd, tdata); + return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim)); } -static void -prof_tdata_expire(tsd_t *tsd, prof_tdata_t *tdata) +static bool +prof_tdata_expire(prof_tdata_t *tdata) { + bool destroy_tdata; - prof_tdata_state_transition(tsd, tdata, prof_tdata_state_expired); + malloc_mutex_lock(tdata->lock); + if (!tdata->expired) { + tdata->expired = true; + destroy_tdata = tdata->attached ? false : + prof_tdata_should_destroy(tdata); + } else + destroy_tdata = false; + malloc_mutex_unlock(tdata->lock); + + return (destroy_tdata); } static prof_tdata_t * prof_tdata_reset_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) { - tsd_t *tsd = (tsd_t *)arg; - prof_tdata_expire(tsd, tdata); - return (NULL); + return (prof_tdata_expire(tdata) ? tdata : NULL); } void prof_reset(tsd_t *tsd, size_t lg_sample) { + prof_tdata_t *next; assert(lg_sample < (sizeof(uint64_t) << 3)); @@ -1664,7 +1738,18 @@ prof_reset(tsd_t *tsd, size_t lg_sample) malloc_mutex_lock(&tdatas_mtx); lg_prof_sample = lg_sample; - tdata_tree_iter(&tdatas, NULL, prof_tdata_reset_iter, tsd); + + next = NULL; + do { + prof_tdata_t *to_destroy = tdata_tree_iter(&tdatas, next, + prof_tdata_reset_iter, NULL); + if (to_destroy != NULL) { + next = tdata_tree_next(&tdatas, to_destroy); + tdata_tree_remove(&tdatas, to_destroy); + prof_tdata_destroy(tsd, to_destroy); + } else + next = NULL; + } while (next != NULL); malloc_mutex_unlock(&tdatas_mtx); malloc_mutex_unlock(&prof_dump_mtx); diff --git a/test/unit/prof_reset.c b/test/unit/prof_reset.c new file mode 100644 index 00000000..73fda419 --- /dev/null +++ b/test/unit/prof_reset.c @@ -0,0 +1,238 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = + "prof:true,prof_active:false,lg_prof_sample:0"; +#endif + +static int +prof_dump_open_intercept(bool propagate_err, const char *filename) +{ + int fd; + + fd = open("/dev/null", O_WRONLY); + assert_d_ne(fd, -1, "Unexpected open() failure"); + + return (fd); +} + +TEST_BEGIN(test_prof_reset_basic) +{ + size_t lg_prof_sample_orig, lg_prof_sample, lg_prof_sample_next; + size_t sz; + unsigned i; + + sz = sizeof(size_t); + assert_d_eq(mallctl("opt.lg_prof_sample", &lg_prof_sample_orig, &sz, + NULL, 0), 0, + "Unexpected mallctl failure while reading profiling sample rate"); + assert_zu_eq(lg_prof_sample_orig, 0, + "Unexpected profiling sample rate"); + sz = sizeof(size_t); + assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, NULL, 0), 0, + "Unexpected mallctl failure while reading profiling sample rate"); + assert_zu_eq(lg_prof_sample_orig, lg_prof_sample, + "Unexpected disagreement between \"opt.lg_prof_sample\" and " + "\"prof.lg_sample\""); + + /* Test simple resets. */ + for (i = 0; i < 2; i++) { + assert_d_eq(mallctl("prof.reset", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure while resetting profile data"); + sz = sizeof(size_t); + assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, + NULL, 0), 0, "Unexpected mallctl failure while reading " + "profiling sample rate"); + assert_zu_eq(lg_prof_sample_orig, lg_prof_sample, + "Unexpected profile sample rate change"); + } + + /* Test resets with prof.lg_sample changes. */ + lg_prof_sample_next = 1; + for (i = 0; i < 2; i++) { + assert_d_eq(mallctl("prof.reset", NULL, NULL, + &lg_prof_sample_next, sizeof(size_t)), 0, + "Unexpected mallctl failure while resetting profile data"); + sz = sizeof(size_t); + assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, + NULL, 0), 0, "Unexpected mallctl failure while reading " + "profiling sample rate"); + assert_zu_eq(lg_prof_sample, lg_prof_sample_next, + "Expected profile sample rate change"); + lg_prof_sample_next = lg_prof_sample_orig; + } + + /* Make sure the test code restored prof.lg_sample. */ + sz = sizeof(size_t); + assert_d_eq(mallctl("prof.lg_sample", &lg_prof_sample, &sz, NULL, 0), 0, + "Unexpected mallctl failure while reading profiling sample rate"); + assert_zu_eq(lg_prof_sample_orig, lg_prof_sample, + "Unexpected disagreement between \"opt.lg_prof_sample\" and " + "\"prof.lg_sample\""); +} +TEST_END + +bool prof_dump_header_intercepted = false; +prof_cnt_t cnt_all_copy = {0, 0, 0, 0}; +static bool +prof_dump_header_intercept(bool propagate_err, const prof_cnt_t *cnt_all) +{ + + prof_dump_header_intercepted = true; + memcpy(&cnt_all_copy, cnt_all, sizeof(prof_cnt_t)); + + return (false); +} + +TEST_BEGIN(test_prof_reset_cleanup) +{ + bool active; + void *p; + prof_dump_header_t *prof_dump_header_orig; + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + // XXX Verify that reset actually drops backtrace count to 0. Alloc an + // object, reset, check bt count, free. prof_bt_count() doesn't do the + // right thing; we need to iterate during dump and count backtraces. + // Or, just intercept prof_dump_header(), which has enough information + // for these purposes. + + assert_zu_eq(prof_bt_count(), 0, "Expected 0 backtraces"); + p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + assert_zu_eq(prof_bt_count(), 1, "Expected 1 backtrace"); + + prof_dump_header_orig = prof_dump_header; + prof_dump_header = prof_dump_header_intercept; + assert_false(prof_dump_header_intercepted, "Unexpected intercept"); + + assert_d_eq(mallctl("prof.dump", NULL, NULL, NULL, 0), + 0, "Unexpected error while dumping heap profile"); + assert_true(prof_dump_header_intercepted, "Expected intercept"); + assert_u64_eq(cnt_all_copy.curobjs, 1, "Expected 1 allocation"); + + assert_d_eq(mallctl("prof.reset", NULL, NULL, NULL, 0), 0, + "Unexpected error while resetting heap profile data"); + assert_d_eq(mallctl("prof.dump", NULL, NULL, NULL, 0), + 0, "Unexpected error while dumping heap profile"); + assert_u64_eq(cnt_all_copy.curobjs, 0, "Expected 0 allocations"); + assert_zu_eq(prof_bt_count(), 1, "Expected 1 backtrace"); + + prof_dump_header = prof_dump_header_orig; + + dallocx(p, 0); + assert_zu_eq(prof_bt_count(), 0, "Expected 0 backtraces"); + + active = false; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while deactivating profiling"); +} +TEST_END + +#define NTHREADS 4 +#define NALLOCS_PER_THREAD (1U << 13) +#define OBJ_RING_BUF_COUNT 1531 +#define RESET_INTERVAL (1U << 10) +#define DUMP_INTERVAL 3677 +static void * +thd_start(void *varg) +{ + unsigned thd_ind = *(unsigned *)varg; + unsigned i; + void *objs[OBJ_RING_BUF_COUNT]; + + memset(objs, 0, sizeof(objs)); + + for (i = 0; i < NALLOCS_PER_THREAD; i++) { + if (i % RESET_INTERVAL == 0) { + assert_d_eq(mallctl("prof.reset", NULL, NULL, NULL, 0), + 0, "Unexpected error while resetting heap profile " + "data"); + } + + if (i % DUMP_INTERVAL == 0) { + assert_d_eq(mallctl("prof.dump", NULL, NULL, NULL, 0), + 0, "Unexpected error while dumping heap profile"); + } + + { + void **pp = &objs[i % OBJ_RING_BUF_COUNT]; + if (*pp != NULL) { + dallocx(*pp, 0); + *pp = NULL; + } + *pp = btalloc(1, thd_ind*NALLOCS_PER_THREAD + i); + assert_ptr_not_null(*pp, + "Unexpected btalloc() failure"); + } + } + + /* Clean up any remaining objects. */ + for (i = 0; i < OBJ_RING_BUF_COUNT; i++) { + void **pp = &objs[i % OBJ_RING_BUF_COUNT]; + if (*pp != NULL) { + dallocx(*pp, 0); + *pp = NULL; + } + } + + return (NULL); +} + +TEST_BEGIN(test_prof_reset) +{ + bool active; + thd_t thds[NTHREADS]; + unsigned thd_args[NTHREADS]; + unsigned i; + size_t bt_count, tdata_count; + + test_skip_if(!config_prof); + + bt_count = prof_bt_count(); + assert_zu_eq(bt_count, 0, + "Unexpected pre-existing tdata structures"); + tdata_count = prof_tdata_count(); + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + for (i = 0; i < NTHREADS; i++) { + thd_args[i] = i; + thd_create(&thds[i], thd_start, (void *)&thd_args[i]); + } + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); + + assert_zu_eq(prof_bt_count(), bt_count, + "Unexpected bactrace count change"); + assert_zu_eq(prof_tdata_count(), tdata_count, + "Unexpected remaining tdata structures"); + + active = false; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while deactivating profiling"); +} +TEST_END +#undef NTHREADS +#undef NALLOCS_PER_THREAD +#undef OBJ_RING_BUF_COUNT +#undef RESET_INTERVAL +#undef DUMP_INTERVAL + +int +main(void) +{ + + /* Intercept dumping prior to running any tests. */ + prof_dump_open = prof_dump_open_intercept; + + return (test( + test_prof_reset_basic, + test_prof_reset_cleanup, + test_prof_reset)); +} From ebbd0c91f0935421c04d05c8bdc6e38762a1e561 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 2 Oct 2014 23:05:23 -0700 Subject: [PATCH 124/352] Remove obsolete comment. --- test/unit/prof_reset.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/unit/prof_reset.c b/test/unit/prof_reset.c index 73fda419..62a4d5af 100644 --- a/test/unit/prof_reset.c +++ b/test/unit/prof_reset.c @@ -94,12 +94,6 @@ TEST_BEGIN(test_prof_reset_cleanup) assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), 0, "Unexpected mallctl failure while activating profiling"); - // XXX Verify that reset actually drops backtrace count to 0. Alloc an - // object, reset, check bt count, free. prof_bt_count() doesn't do the - // right thing; we need to iterate during dump and count backtraces. - // Or, just intercept prof_dump_header(), which has enough information - // for these purposes. - assert_zu_eq(prof_bt_count(), 0, "Expected 0 backtraces"); p = mallocx(1, 0); assert_ptr_not_null(p, "Unexpected mallocx() failure"); From 551ebc43647521bdd0bc78558b106762b3388928 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 3 Oct 2014 10:16:09 -0700 Subject: [PATCH 125/352] Convert to uniform style: cond == false --> !cond --- include/jemalloc/internal/arena.h | 11 +++--- include/jemalloc/internal/bitmap.h | 8 ++-- .../jemalloc/internal/jemalloc_internal.h.in | 2 +- include/jemalloc/internal/prof.h | 2 +- include/jemalloc/internal/rb.h | 7 ++-- include/jemalloc/internal/tcache.h | 8 ++-- src/arena.c | 28 +++++++------- src/chunk.c | 16 ++++---- src/chunk_dss.c | 4 +- src/chunk_mmap.c | 4 +- src/ckh.c | 10 ++--- src/ctl.c | 37 +++++++++---------- src/huge.c | 8 ++-- src/jemalloc.c | 22 +++++------ src/prof.c | 30 +++++++-------- src/stats.c | 2 +- src/tcache.c | 8 ++-- src/util.c | 12 +++--- test/unit/ckh.c | 3 +- test/unit/rb.c | 4 +- 20 files changed, 111 insertions(+), 115 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 48fd2055..2e9920ce 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -1111,13 +1111,12 @@ arena_salloc(const void *ptr, bool demote) pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; assert(arena_mapbits_allocated_get(chunk, pageind) != 0); binind = arena_mapbits_binind_get(chunk, pageind); - if (unlikely(binind == BININD_INVALID || (config_prof && demote == false - && arena_mapbits_large_get(chunk, pageind) != 0))) { + if (unlikely(binind == BININD_INVALID || (config_prof && !demote && + arena_mapbits_large_get(chunk, pageind) != 0))) { /* - * Large allocation. In the common case (demote == true), and - * as this is an inline function, most callers will only end up - * looking at binind to determine that ptr is a small - * allocation. + * Large allocation. In the common case (demote), and as this + * is an inline function, most callers will only end up looking + * at binind to determine that ptr is a small allocation. */ assert(((uintptr_t)ptr & PAGE_MASK) == 0); ret = arena_mapbits_large_size_get(chunk, pageind); diff --git a/include/jemalloc/internal/bitmap.h b/include/jemalloc/internal/bitmap.h index 4ca40ffd..fcc6005c 100644 --- a/include/jemalloc/internal/bitmap.h +++ b/include/jemalloc/internal/bitmap.h @@ -139,7 +139,7 @@ bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) bitmap_t g; assert(bit < binfo->nbits); - assert(bitmap_get(bitmap, binfo, bit) == false); + assert(!bitmap_get(bitmap, binfo, bit)); goff = bit >> LG_BITMAP_GROUP_NBITS; gp = &bitmap[goff]; g = *gp; @@ -172,7 +172,7 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) bitmap_t g; unsigned i; - assert(bitmap_full(bitmap, binfo) == false); + assert(!bitmap_full(bitmap, binfo)); i = binfo->nlevels - 1; g = bitmap[binfo->levels[i].group_offset]; @@ -204,7 +204,7 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) == 0); g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); *gp = g; - assert(bitmap_get(bitmap, binfo, bit) == false); + assert(!bitmap_get(bitmap, binfo, bit)); /* Propagate group state transitions up the tree. */ if (propagate) { unsigned i; @@ -218,7 +218,7 @@ bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) == 0); g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); *gp = g; - if (propagate == false) + if (!propagate) break; } } diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index bff2bd27..ed25172f 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -714,7 +714,7 @@ isalloc(const void *ptr, bool demote) assert(ptr != NULL); /* Demotion only makes sense if config_prof is true. */ - assert(config_prof || demote == false); + assert(config_prof || !demote); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 91c871de..ea52a63b 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -388,7 +388,7 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, /* Compute new sample threshold. */ if (update) prof_sample_threshold_update(tdata); - return (tdata->active == false); + return (!tdata->active); } } diff --git a/include/jemalloc/internal/rb.h b/include/jemalloc/internal/rb.h index ffe3bb0d..64fab89c 100644 --- a/include/jemalloc/internal/rb.h +++ b/include/jemalloc/internal/rb.h @@ -593,7 +593,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \ if (left != &rbtree->rbt_nil) { \ /* node has no successor, but it has a left child. */\ /* Splice node out, without losing the left child. */\ - assert(rbtn_red_get(a_type, a_field, node) == false); \ + assert(!rbtn_red_get(a_type, a_field, node)); \ assert(rbtn_red_get(a_type, a_field, left)); \ rbtn_black_set(a_type, a_field, left); \ if (pathp == path) { \ @@ -629,8 +629,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \ if (pathp->cmp < 0) { \ rbtn_left_set(a_type, a_field, pathp->node, \ pathp[1].node); \ - assert(rbtn_red_get(a_type, a_field, pathp[1].node) \ - == false); \ + assert(!rbtn_red_get(a_type, a_field, pathp[1].node)); \ if (rbtn_red_get(a_type, a_field, pathp->node)) { \ a_type *right = rbtn_right_get(a_type, a_field, \ pathp->node); \ @@ -862,7 +861,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \ } \ /* Set root. */ \ rbtree->rbt_root = path->node; \ - assert(rbtn_red_get(a_type, a_field, rbtree->rbt_root) == false); \ + assert(!rbtn_red_get(a_type, a_field, rbtree->rbt_root)); \ } \ a_attr a_type * \ a_prefix##iter_recurse(a_rbt_type *rbtree, a_type *node, \ diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 6804668f..bc0b41c7 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -191,9 +191,9 @@ tcache_get(tsd_t *tsd, bool create) { tcache_t *tcache; - if (config_tcache == false) + if (!config_tcache) return (NULL); - if (config_lazy_lock && isthreaded == false) + if (config_lazy_lock && !isthreaded) return (NULL); /* * If create is true, the caller has already assured that tsd is @@ -261,7 +261,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) } assert(tcache_salloc(ret) == size); - if (likely(zero == false)) { + if (likely(!zero)) { if (config_fill) { if (unlikely(opt_junk)) { arena_alloc_junk_small(ret, @@ -315,7 +315,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) arena_mapbits_large_binind_set(chunk, pageind, BININD_INVALID); } - if (likely(zero == false)) { + if (likely(!zero)) { if (config_fill) { if (unlikely(opt_junk)) memset(ret, 0xa5, size); diff --git a/src/arena.c b/src/arena.c index ef391b16..79fea728 100644 --- a/src/arena.c +++ b/src/arena.c @@ -178,7 +178,7 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) void *rpages; assert(run->nfree > 0); - assert(bitmap_full(run->bitmap, &bin_info->bitmap_info) == false); + assert(!bitmap_full(run->bitmap, &bin_info->bitmap_info)); regind = bitmap_sfu(run->bitmap, &bin_info->bitmap_info); miscelm = arena_run_to_miscelm(run); @@ -524,7 +524,7 @@ arena_chunk_init_hard(arena_t *arena) * There is no need to initialize the internal page map entries unless * the chunk is not zeroed. */ - if (zero == false) { + if (!zero) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( (void *)arena_bitselm_get(chunk, map_bias+1), (size_t)((uintptr_t) arena_bitselm_get(chunk, @@ -782,7 +782,7 @@ arena_compute_npurge(arena_t *arena, bool all) * Compute the minimum number of pages that this thread should try to * purge. */ - if (all == false) { + if (!all) { size_t threshold = (arena->nactive >> opt_lg_dirty_mult); npurge = arena->ndirty - threshold; @@ -829,7 +829,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, nstashed += npages; - if (all == false && nstashed >= npurge) + if (!all && nstashed >= npurge) break; } @@ -1049,7 +1049,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) */ assert(arena_mapbits_dirty_get(chunk, run_ind) == arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); - if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) + if (!cleaned && arena_mapbits_dirty_get(chunk, run_ind) != 0) dirty = true; flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; @@ -1481,10 +1481,10 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) bin->stats.nrequests++; } malloc_mutex_unlock(&bin->lock); - if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) + if (config_prof && !isthreaded && arena_prof_accum(arena, size)) prof_idump(); - if (zero == false) { + if (!zero) { if (config_fill) { if (unlikely(opt_junk)) { arena_alloc_junk_small(ret, @@ -1537,7 +1537,7 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) if (config_prof && idump) prof_idump(); - if (zero == false) { + if (!zero) { if (config_fill) { if (unlikely(opt_junk)) memset(ret, 0xa5, size); @@ -1608,7 +1608,7 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) } malloc_mutex_unlock(&arena->lock); - if (config_fill && zero == false) { + if (config_fill && !zero) { if (unlikely(opt_junk)) memset(ret, 0xa5, size); else if (unlikely(opt_zero)) @@ -2008,7 +2008,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, PAGE_CEILING(size), psize - PAGE_CEILING(size), zero); - if (config_fill && ret == false && zero == false) { + if (config_fill && !ret && !zero) { if (unlikely(opt_junk)) { memset((void *)((uintptr_t)ptr + oldsize), 0xa5, isalloc(ptr, @@ -2044,8 +2044,8 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, } else { assert(size <= arena_maxclass); if (size + extra > SMALL_MAXCLASS) { - if (arena_ralloc_large(ptr, oldsize, size, - extra, zero) == false) + if (!arena_ralloc_large(ptr, oldsize, size, + extra, zero)) return (false); } } @@ -2064,7 +2064,7 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t copysize; /* Try to avoid moving the allocation. */ - if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) + if (!arena_ralloc_no_move(ptr, oldsize, size, extra, zero)) return (ptr); /* @@ -2130,7 +2130,7 @@ bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) { - if (have_dss == false) + if (!have_dss) return (dss_prec != dss_prec_disabled); malloc_mutex_lock(&arena->lock); arena->dss_prec = dss_prec; diff --git a/src/chunk.c b/src/chunk.c index 874002cf..cde8606e 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -121,7 +121,7 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, if (node != NULL) base_node_dalloc(node); if (*zero) { - if (zeroed == false) + if (!zeroed) memset(ret, 0, size); else if (config_debug) { size_t i; @@ -136,10 +136,10 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, } /* - * If the caller specifies (*zero == false), it is still possible to receive - * zeroed memory, in which case *zero is toggled to true. arena_chunk_alloc() - * takes advantage of this to avoid demanding zeroed chunks, but taking - * advantage of them if they are returned. + * If the caller specifies (!*zero), it is still possible to receive zeroed + * memory, in which case *zero is toggled to true. arena_chunk_alloc() takes + * advantage of this to avoid demanding zeroed chunks, but taking advantage of + * them if they are returned. */ static void * chunk_alloc_core(size_t size, size_t alignment, bool base, bool *zero, @@ -186,7 +186,7 @@ chunk_register(void *chunk, size_t size, bool base) assert(chunk != NULL); assert(CHUNK_ADDR2BASE(chunk) == chunk); - if (config_ivsalloc && base == false) { + if (config_ivsalloc && !base) { if (rtree_set(chunks_rtree, (uintptr_t)chunk, 1)) return (true); } @@ -288,7 +288,7 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, extent_tree_szad_remove(chunks_szad, node); node->addr = chunk; node->size += size; - node->zeroed = (node->zeroed && (unzeroed == false)); + node->zeroed = (node->zeroed && !unzeroed); extent_tree_szad_insert(chunks_szad, node); } else { /* Coalescing forward failed, so insert a new node. */ @@ -305,7 +305,7 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, xnode = NULL; /* Prevent deallocation below. */ node->addr = chunk; node->size = size; - node->zeroed = (unzeroed == false); + node->zeroed = !unzeroed; extent_tree_ad_insert(chunks_ad, node); extent_tree_szad_insert(chunks_szad, node); } diff --git a/src/chunk_dss.c b/src/chunk_dss.c index 82faf918..cce71041 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -45,7 +45,7 @@ chunk_dss_prec_get(void) { dss_prec_t ret; - if (have_dss == false) + if (!have_dss) return (dss_prec_disabled); malloc_mutex_lock(&dss_mtx); ret = dss_prec_default; @@ -57,7 +57,7 @@ bool chunk_dss_prec_set(dss_prec_t dss_prec) { - if (have_dss == false) + if (!have_dss) return (dss_prec != dss_prec_disabled); malloc_mutex_lock(&dss_mtx); dss_prec_default = dss_prec; diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c index 65137b41..7e02c102 100644 --- a/src/chunk_mmap.c +++ b/src/chunk_mmap.c @@ -132,7 +132,7 @@ pages_purge(void *addr, size_t length) # error "No madvise(2) flag defined for purging unused dirty pages." # endif int err = madvise(addr, length, JEMALLOC_MADV_PURGE); - unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0); + unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); # undef JEMALLOC_MADV_PURGE # undef JEMALLOC_MADV_ZEROS #else @@ -209,5 +209,5 @@ chunk_dalloc_mmap(void *chunk, size_t size) if (config_munmap) pages_unmap(chunk, size); - return (config_munmap == false); + return (!config_munmap); } diff --git a/src/ckh.c b/src/ckh.c index 7c7cc098..3a545966 100644 --- a/src/ckh.c +++ b/src/ckh.c @@ -185,7 +185,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, } bucket = tbucket; - if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) + if (!ckh_try_bucket_insert(ckh, bucket, key, data)) return (false); } } @@ -201,12 +201,12 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) /* Try to insert in primary bucket. */ bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1); - if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) + if (!ckh_try_bucket_insert(ckh, bucket, key, data)) return (false); /* Try to insert in secondary bucket. */ bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); - if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) + if (!ckh_try_bucket_insert(ckh, bucket, key, data)) return (false); /* @@ -281,7 +281,7 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) tab = ttab; ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; - if (ckh_rebuild(ckh, tab) == false) { + if (!ckh_rebuild(ckh, tab)) { idalloc(tsd, tab); break; } @@ -327,7 +327,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) tab = ttab; ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; - if (ckh_rebuild(ckh, tab) == false) { + if (!ckh_rebuild(ckh, tab)) { idalloc(tsd, tab); #ifdef CKH_COUNT ckh->nshrinks++; diff --git a/src/ctl.c b/src/ctl.c index c55f6e44..b85710c0 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -36,8 +36,7 @@ static inline const ctl_indexed_node_t * ctl_indexed_node(const ctl_node_t *node) { - return ((node->named == false) ? (const ctl_indexed_node_t *)node : - NULL); + return (!node->named ? (const ctl_indexed_node_t *)node : NULL); } /******************************************************************************/ @@ -693,7 +692,7 @@ ctl_init(void) bool ret; malloc_mutex_lock(&ctl_mtx); - if (ctl_initialized == false) { + if (!ctl_initialized) { /* * Allocate space for one extra arena stats element, which * contains summed stats across all arenas. @@ -843,7 +842,7 @@ ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t mib[CTL_MAX_DEPTH]; const ctl_named_node_t *node; - if (ctl_initialized == false && ctl_init()) { + if (!ctl_initialized && ctl_init()) { ret = EAGAIN; goto label_return; } @@ -870,7 +869,7 @@ ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp) { int ret; - if (ctl_initialized == false && ctl_init()) { + if (!ctl_initialized && ctl_init()) { ret = EAGAIN; goto label_return; } @@ -888,7 +887,7 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, const ctl_named_node_t *node; size_t i; - if (ctl_initialized == false && ctl_init()) { + if (!ctl_initialized && ctl_init()) { ret = EAGAIN; goto label_return; } @@ -1015,7 +1014,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ int ret; \ t oldval; \ \ - if ((c) == false) \ + if (!(c)) \ return (ENOENT); \ if (l) \ malloc_mutex_lock(&ctl_mtx); \ @@ -1038,7 +1037,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ int ret; \ t oldval; \ \ - if ((c) == false) \ + if (!(c)) \ return (ENOENT); \ malloc_mutex_lock(&ctl_mtx); \ READONLY(); \ @@ -1082,7 +1081,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ int ret; \ t oldval; \ \ - if ((c) == false) \ + if (!(c)) \ return (ENOENT); \ READONLY(); \ oldval = (v); \ @@ -1119,7 +1118,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ t oldval; \ tsd_t *tsd; \ \ - if ((c) == false) \ + if (!(c)) \ return (ENOENT); \ READONLY(); \ tsd = tsd_tryget(); \ @@ -1291,7 +1290,7 @@ thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, int ret; bool oldval; - if (config_tcache == false) + if (!config_tcache) return (ENOENT); oldval = tcache_enabled_get(); @@ -1315,7 +1314,7 @@ thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, { int ret; - if (config_tcache == false) + if (!config_tcache) return (ENOENT); READONLY(); @@ -1335,7 +1334,7 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp, int ret; const char *oldname; - if (config_prof == false) + if (!config_prof) return (ENOENT); oldname = prof_thread_name_get(); @@ -1372,7 +1371,7 @@ thread_prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, int ret; bool oldval; - if (config_prof == false) + if (!config_prof) return (ENOENT); oldval = prof_thread_active_get(); @@ -1459,7 +1458,7 @@ arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, } } - if (match == false) { + if (!match) { ret = EINVAL; goto label_return; } @@ -1668,7 +1667,7 @@ prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; bool oldval; - if (config_prof == false) + if (!config_prof) return (ENOENT); malloc_mutex_lock(&ctl_mtx); /* Protect opt_prof_active. */ @@ -1697,7 +1696,7 @@ prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; const char *filename = NULL; - if (config_prof == false) + if (!config_prof) return (ENOENT); WRITEONLY(); @@ -1721,7 +1720,7 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, size_t lg_sample = lg_prof_sample; tsd_t *tsd; - if (config_prof == false) + if (!config_prof) return (ENOENT); WRITEONLY(); @@ -1847,7 +1846,7 @@ stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) const ctl_named_node_t * ret; malloc_mutex_lock(&ctl_mtx); - if (i > ctl_stats.narenas || ctl_stats.arenas[i].initialized == false) { + if (i > ctl_stats.narenas || !ctl_stats.arenas[i].initialized) { ret = NULL; goto label_return; } diff --git a/src/huge.c b/src/huge.c index 40d1362d..2f059b4d 100644 --- a/src/huge.c +++ b/src/huge.c @@ -62,10 +62,10 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, extent_tree_ad_insert(&huge, node); malloc_mutex_unlock(&huge_mtx); - if (config_fill && zero == false) { + if (config_fill && !zero) { if (unlikely(opt_junk)) memset(ret, 0xa5, csize); - else if (unlikely(opt_zero) && is_zeroed == false) + else if (unlikely(opt_zero) && !is_zeroed) memset(ret, 0, csize); } @@ -85,7 +85,7 @@ huge_dalloc_junk(void *ptr, size_t usize) * Only bother junk filling if the chunk isn't about to be * unmapped. */ - if (config_munmap == false || (have_dss && chunk_in_dss(ptr))) + if (!config_munmap || (have_dss && chunk_in_dss(ptr))) memset(ptr, 0x5a, usize); } } @@ -156,7 +156,7 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t copysize; /* Try to avoid moving the allocation. */ - if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false) + if (!huge_ralloc_no_move(ptr, oldsize, size, extra)) return (ptr); /* diff --git a/src/jemalloc.c b/src/jemalloc.c index 3012f558..0d041313 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -119,7 +119,7 @@ arenas_extend(unsigned ind) arena_t *ret; ret = (arena_t *)base_alloc(sizeof(arena_t)); - if (ret != NULL && arena_new(ret, ind) == false) { + if (ret != NULL && !arena_new(ret, ind)) { arenas[ind] = ret; return (ret); } @@ -326,7 +326,7 @@ malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, *k_p = opts; - for (accept = false; accept == false;) { + for (accept = false; !accept;) { switch (*opts) { case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': @@ -361,7 +361,7 @@ malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, } } - for (accept = false; accept == false;) { + for (accept = false; !accept;) { switch (*opts) { case ',': opts++; @@ -418,7 +418,7 @@ malloc_conf_init(void) in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; if (config_fill && unlikely(in_valgrind)) { opt_junk = false; - assert(opt_zero == false); + assert(!opt_zero); opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; opt_redzone = true; } @@ -496,8 +496,8 @@ malloc_conf_init(void) opts = buf; } - while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, - &vlen) == false) { + while (*opts != '\0' && !malloc_conf_next(&opts, &k, &klen, &v, + &vlen)) { #define CONF_MATCH(n) \ (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) #define CONF_HANDLE_BOOL(o, n, cont) \ @@ -607,7 +607,7 @@ malloc_conf_init(void) } } } - if (match == false) { + if (!match) { malloc_conf_error("Invalid conf value", k, klen, v, vlen); } @@ -697,13 +697,13 @@ malloc_init_hard(void) return (false); } #ifdef JEMALLOC_THREADED_INIT - if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { + if (malloc_initializer != NO_INITIALIZER && !IS_INITIALIZER) { /* Busy-wait until the initializing thread completes. */ do { malloc_mutex_unlock(&init_lock); CPU_SPINWAIT; malloc_mutex_lock(&init_lock); - } while (malloc_initialized == false); + } while (!malloc_initialized); malloc_mutex_unlock(&init_lock); return (false); } @@ -2011,7 +2011,7 @@ _malloc_prefork(void) unsigned i; #ifdef JEMALLOC_MUTEX_INIT_CB - if (malloc_initialized == false) + if (!malloc_initialized) return; #endif assert(malloc_initialized); @@ -2040,7 +2040,7 @@ _malloc_postfork(void) unsigned i; #ifdef JEMALLOC_MUTEX_INIT_CB - if (malloc_initialized == false) + if (!malloc_initialized) return; #endif assert(malloc_initialized); diff --git a/src/prof.c b/src/prof.c index 0a96d85f..29b4baaa 100644 --- a/src/prof.c +++ b/src/prof.c @@ -232,7 +232,7 @@ prof_enter(prof_tdata_t *tdata) cassert(config_prof); - assert(tdata->enq == false); + assert(!tdata->enq); tdata->enq = true; malloc_mutex_lock(&bt2gctx_mtx); @@ -578,7 +578,7 @@ prof_gctx_should_destroy(prof_gctx_t *gctx) if (opt_prof_accum) return (false); - if (tctx_tree_empty(&gctx->tctxs) == false) + if (!tctx_tree_empty(&gctx->tctxs)) return (false); if (gctx->nlimbo != 0) return (false); @@ -595,7 +595,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) assert(tctx->cnts.curobjs == 0); assert(tctx->cnts.curbytes == 0); - assert(opt_prof_accum == false); + assert(!opt_prof_accum); assert(tctx->cnts.accumobjs == 0); assert(tctx->cnts.accumbytes == 0); @@ -858,7 +858,7 @@ prof_dump_open(bool propagate_err, const char *filename) int fd; fd = creat(filename, 0644); - if (fd == -1 && propagate_err == false) { + if (fd == -1 && !propagate_err) { malloc_printf(": creat(\"%s\"), 0644) failed\n", filename); if (opt_abort) @@ -883,7 +883,7 @@ prof_dump_flush(bool propagate_err) err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); if (err == -1) { - if (propagate_err == false) { + if (!propagate_err) { malloc_write(": write() failed during heap " "profile flush\n"); if (opt_abort) @@ -1145,8 +1145,8 @@ prof_tdata_merge_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) tdata->dumping = true; memset(&tdata->cnt_summed, 0, sizeof(prof_cnt_t)); - for (tabind = 0; ckh_iter(&tdata->bt2tctx, &tabind, NULL, - &tctx.v) == false;) + for (tabind = 0; !ckh_iter(&tdata->bt2tctx, &tabind, NULL, + &tctx.v);) prof_tctx_merge_tdata(tctx.p, tdata); cnt_all->curobjs += tdata->cnt_summed.curobjs; @@ -1167,7 +1167,7 @@ prof_tdata_dump_iter(prof_tdata_tree_t *tdatas, prof_tdata_t *tdata, void *arg) { bool propagate_err = *(bool *)arg; - if (tdata->dumping == false) + if (!tdata->dumping) return (NULL); if (prof_dump_printf(propagate_err, @@ -1220,7 +1220,7 @@ prof_dump_gctx(bool propagate_err, prof_gctx_t *gctx, const prof_bt_t *bt, cassert(config_prof); /* Avoid dumping such gctx's that have no useful data. */ - if ((opt_prof_accum == false && gctx->cnt_summed.curobjs == 0) || + if ((!opt_prof_accum && gctx->cnt_summed.curobjs == 0) || (opt_prof_accum && gctx->cnt_summed.accumobjs == 0)) { assert(gctx->cnt_summed.curobjs == 0); assert(gctx->cnt_summed.curbytes == 0); @@ -1374,7 +1374,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) * summing. */ gctx_tree_new(&gctxs); - for (tabind = 0; ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v) == false;) + for (tabind = 0; !ckh_iter(&bt2gctx, &tabind, NULL, &gctx.v);) prof_dump_gctx_prep(gctx.p, &gctxs); /* @@ -1457,7 +1457,7 @@ prof_fdump(void) cassert(config_prof); - if (prof_booted == false) + if (!prof_booted) return; if ((tsd = tsd_tryget()) == NULL) return; @@ -1479,7 +1479,7 @@ prof_idump(void) cassert(config_prof); - if (prof_booted == false) + if (!prof_booted) return; if ((tsd = tsd_tryget()) == NULL) return; @@ -1508,7 +1508,7 @@ prof_mdump(const char *filename) cassert(config_prof); - if (opt_prof == false || prof_booted == false) + if (!opt_prof || !prof_booted) return (true); if ((tsd = tsd_tryget()) == NULL) return (true); @@ -1535,7 +1535,7 @@ prof_gdump(void) cassert(config_prof); - if (prof_booted == false) + if (!prof_booted) return; if ((tsd = tsd_tryget()) == NULL) return; @@ -1855,7 +1855,7 @@ prof_boot1(void) * initialized, so this function must be executed early. */ - if (opt_prof_leak && opt_prof == false) { + if (opt_prof_leak && !opt_prof) { /* * Enable opt_prof, but in such a way that profiles are never * automatically dumped. diff --git a/src/stats.c b/src/stats.c index db34275e..aa095500 100644 --- a/src/stats.c +++ b/src/stats.c @@ -505,7 +505,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, ninitialized++; } - if (ninitialized > 1 || unmerged == false) { + if (ninitialized > 1 || !unmerged) { /* Print merged arena stats. */ malloc_cprintf(write_cb, cbopaque, "\nMerged arenas stats:\n"); diff --git a/src/tcache.c b/src/tcache.c index bb4c3cc0..6f3408cd 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -101,7 +101,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, malloc_mutex_lock(&bin->lock); if (config_stats && arena == tcache->arena) { - assert(merged_stats == false); + assert(!merged_stats); merged_stats = true; bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; @@ -132,7 +132,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, } malloc_mutex_unlock(&bin->lock); } - if (config_stats && merged_stats == false) { + if (config_stats && !merged_stats) { /* * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. @@ -210,7 +210,7 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, if (config_prof && idump) prof_idump(); } - if (config_stats && merged_stats == false) { + if (config_stats && !merged_stats) { /* * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. @@ -262,7 +262,7 @@ tcache_t * tcache_get_hard(tsd_t *tsd) { - if (tcache_enabled_get() == false) { + if (!tcache_enabled_get()) { tcache_enabled_set(false); /* Memoize. */ return (NULL); } diff --git a/src/util.c b/src/util.c index 1717f08e..bfd86af8 100644 --- a/src/util.c +++ b/src/util.c @@ -266,7 +266,7 @@ d2s(intmax_t x, char sign, char *s, size_t *slen_p) sign = '-'; switch (sign) { case '-': - if (neg == false) + if (!neg) break; /* Fall through. */ case ' ': @@ -329,7 +329,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) /* Left padding. */ \ size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ (size_t)width - slen : 0); \ - if (left_justify == false && pad_len != 0) { \ + if (!left_justify && pad_len != 0) { \ size_t j; \ for (j = 0; j < pad_len; j++) \ APPEND_C(' '); \ @@ -406,19 +406,19 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) while (true) { switch (*f) { case '#': - assert(alt_form == false); + assert(!alt_form); alt_form = true; break; case '-': - assert(left_justify == false); + assert(!left_justify); left_justify = true; break; case ' ': - assert(plus_space == false); + assert(!plus_space); plus_space = true; break; case '+': - assert(plus_plus == false); + assert(!plus_plus); plus_plus = true; break; default: goto label_width; diff --git a/test/unit/ckh.c b/test/unit/ckh.c index 148b81e7..03b4f716 100644 --- a/test/unit/ckh.c +++ b/test/unit/ckh.c @@ -162,8 +162,7 @@ TEST_BEGIN(test_insert_iter_remove) memset(seen, 0, sizeof(seen)); - for (tabind = 0; ckh_iter(&ckh, &tabind, &q, &r) == - false;) { + for (tabind = 0; !ckh_iter(&ckh, &tabind, &q, &r);) { size_t k; assert_ptr_eq(q, r, "Key and val not equal"); diff --git a/test/unit/rb.c b/test/unit/rb.c index e43907f1..b38eb0e3 100644 --- a/test/unit/rb.c +++ b/test/unit/rb.c @@ -5,7 +5,7 @@ for (rbp_bh_t = (a_rbt)->rbt_root, (r_height) = 0; \ rbp_bh_t != &(a_rbt)->rbt_nil; \ rbp_bh_t = rbtn_left_get(a_type, a_field, rbp_bh_t)) { \ - if (rbtn_red_get(a_type, a_field, rbp_bh_t) == false) { \ + if (!rbtn_red_get(a_type, a_field, rbp_bh_t)) { \ (r_height)++; \ } \ } \ @@ -75,7 +75,7 @@ tree_recurse(node_t *node, unsigned black_height, unsigned black_depth, node_t *left_node = rbtn_left_get(node_t, link, node); node_t *right_node = rbtn_right_get(node_t, link, node); - if (rbtn_red_get(node_t, link, node) == false) + if (!rbtn_red_get(node_t, link, node)) black_depth++; /* Red nodes must be interleaved with black nodes. */ From fc12c0b8bc1160530d1e3e641b76d2a4f793136f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 3 Oct 2014 23:25:30 -0700 Subject: [PATCH 126/352] Implement/test/fix prof-related mallctl's. Implement/test/fix the opt.prof_thread_active_init, prof.thread_active_init, and thread.prof.active mallctl's. Test/fix the thread.prof.name mallctl. Refactor opt_prof_active to be read-only and move mutable state into the prof_active variable. Stop leaning on ctl-related locking for protection. --- Makefile.in | 2 + doc/jemalloc.xml.in | 52 ++++++- include/jemalloc/internal/private_symbols.txt | 5 + include/jemalloc/internal/prof.h | 34 +++-- src/ctl.c | 73 ++++++--- src/jemalloc.c | 2 + src/prof.c | 142 +++++++++++++++--- src/stats.c | 33 ++-- test/unit/prof_active.c | 136 +++++++++++++++++ test/unit/prof_reset.c | 4 + test/unit/prof_thread_name.c | 128 ++++++++++++++++ 11 files changed, 545 insertions(+), 66 deletions(-) create mode 100644 test/unit/prof_active.c create mode 100644 test/unit/prof_thread_name.c diff --git a/Makefile.in b/Makefile.in index 5267bea4..52f5a9d2 100644 --- a/Makefile.in +++ b/Makefile.in @@ -123,9 +123,11 @@ TESTS_UNIT := $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/mq.c \ $(srcroot)test/unit/mtx.c \ $(srcroot)test/unit/prof_accum.c \ + $(srcroot)test/unit/prof_active.c \ $(srcroot)test/unit/prof_gdump.c \ $(srcroot)test/unit/prof_idump.c \ $(srcroot)test/unit/prof_reset.c \ + $(srcroot)test/unit/prof_thread_name.c \ $(srcroot)test/unit/ql.c \ $(srcroot)test/unit/qr.c \ $(srcroot)test/unit/quarantine.c \ diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index b586e690..6abb50bc 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1061,6 +1061,21 @@ malloc_conf = "xmalloc:true";]]> This option is enabled by default. + + + opt.prof_thread_active_init + (bool) + r- + [] + + Initial setting for thread.prof.active + in newly created threads. The initial setting for newly created threads + can also be changed during execution via the prof.thread_active_init + mallctl. This option is enabled by default. + + opt.lg_prof_sample @@ -1264,7 +1279,8 @@ malloc_conf = "xmalloc:true";]]> thread.prof.name (const char *) - rw + r- or + -w [] Get/set the descriptive name associated with the calling @@ -1272,7 +1288,15 @@ malloc_conf = "xmalloc:true";]]> created, so the input string need not be maintained after this interface completes execution. The output string of this interface should be copied for non-ephemeral uses, because multiple implementation details - can cause asynchronous string deallocation. + can cause asynchronous string deallocation. Furthermore, each + invocation of this interface can only read or write; simultaneous + read/write is not supported due to string lifetime limitations. The + name string must nil-terminated and comprised only of characters in the + sets recognized + by isgraph + 3 and + isblank + 3. @@ -1283,7 +1307,7 @@ malloc_conf = "xmalloc:true";]]> [] Control whether sampling is currently active for the - calling thread. This is a deactivation mechanism in addition to prof.active; both must be active for the calling thread to sample. This flag is enabled by default. @@ -1508,6 +1532,20 @@ malloc_conf = "xmalloc:true";]]> and returning the new arena index. + + + prof.thread_active_init + (bool) + rw + [] + + Control the initial setting for thread.prof.active + in newly created threads. See the opt.prof_thread_active_init + option for additional information. + + prof.active @@ -1518,8 +1556,9 @@ malloc_conf = "xmalloc:true";]]> Control whether sampling is currently active. See the opt.prof_active - option for additional information. - + option for additional information, as well as the interrelated thread.prof.active + mallctl. @@ -1548,7 +1587,8 @@ malloc_conf = "xmalloc:true";]]> Reset all memory profile statistics, and optionally update the sample rate (see opt.lg_prof_sample - and prof.lg_sample). + and prof.lg_sample). diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 33f8ce01..63657833 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -285,6 +285,9 @@ opt_zero p2rz pages_purge pow2_ceil +prof_active_get +prof_active_get_unlocked +prof_active_set prof_alloc_prep prof_alloc_rollback prof_backtrace @@ -316,6 +319,8 @@ prof_tdata_cleanup prof_tdata_get prof_tdata_init prof_thread_active_get +prof_thread_active_init_get +prof_thread_active_init_set prof_thread_active_set prof_thread_name_get prof_thread_name_set diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index ea52a63b..3d3f8f4e 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -215,13 +215,8 @@ typedef rb_tree(prof_tdata_t) prof_tdata_tree_t; #ifdef JEMALLOC_H_EXTERNS extern bool opt_prof; -/* - * Even if opt_prof is true, sampling can be temporarily disabled by setting - * opt_prof_active to false. No locking is used when updating opt_prof_active, - * so there are no guarantees regarding how long it will take for all threads - * to notice state changes. - */ extern bool opt_prof_active; +extern bool opt_prof_thread_active_init; extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ extern bool opt_prof_gdump; /* High-water memory dumping. */ @@ -235,6 +230,9 @@ extern char opt_prof_prefix[ #endif 1]; +/* Accessed via prof_active_[gs]et{_unlocked,}(). */ +extern bool prof_active; + /* * Profile dump interval, measured in bytes allocated. Each arena triggers a * profile dump when it reaches this threshold. The effect is that the @@ -274,9 +272,13 @@ prof_tdata_t *prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata); void prof_reset(tsd_t *tsd, size_t lg_sample); void prof_tdata_cleanup(tsd_t *tsd); const char *prof_thread_name_get(void); -bool prof_thread_name_set(tsd_t *tsd, const char *thread_name); +bool prof_active_get(void); +bool prof_active_set(bool active); +int prof_thread_name_set(tsd_t *tsd, const char *thread_name); bool prof_thread_active_get(void); bool prof_thread_active_set(bool active); +bool prof_thread_active_init_get(void); +bool prof_thread_active_init_set(bool active_init); void prof_boot0(void); void prof_boot1(void); bool prof_boot2(void); @@ -290,6 +292,7 @@ void prof_sample_threshold_update(prof_tdata_t *tdata); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +bool prof_active_get_unlocked(void); prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, prof_tdata_t **tdata_out); @@ -305,6 +308,19 @@ void prof_free(tsd_t *tsd, const void *ptr, size_t usize); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) +JEMALLOC_INLINE bool +prof_active_get_unlocked(void) +{ + + /* + * Even if opt_prof is true, sampling can be temporarily disabled by + * setting prof_active to false. No locking is used when reading + * prof_active in the fast path, so there are no guarantees regarding + * how long it will take for all threads to notice state changes. + */ + return (prof_active); +} + JEMALLOC_INLINE prof_tdata_t * prof_tdata_get(tsd_t *tsd, bool create) { @@ -401,8 +417,8 @@ prof_alloc_prep(tsd_t *tsd, size_t usize, bool update) assert(usize == s2u(usize)); - if (!opt_prof_active || likely(prof_sample_accum_update(tsd, usize, - update, &tdata))) + if (!prof_active_get_unlocked() || likely(prof_sample_accum_update(tsd, + usize, update, &tdata))) ret = (prof_tctx_t *)(uintptr_t)1U; else { bt_init(&bt, tdata->vec); diff --git a/src/ctl.c b/src/ctl.c index b85710c0..8f9faa56 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -7,7 +7,6 @@ /* * ctl_mtx protects the following: * - ctl_stats.* - * - opt_prof_active */ static malloc_mutex_t ctl_mtx; static bool ctl_initialized; @@ -104,6 +103,7 @@ CTL_PROTO(opt_lg_tcache_max) CTL_PROTO(opt_prof) CTL_PROTO(opt_prof_prefix) CTL_PROTO(opt_prof_active) +CTL_PROTO(opt_prof_thread_active_init) CTL_PROTO(opt_lg_prof_sample) CTL_PROTO(opt_lg_prof_interval) CTL_PROTO(opt_prof_gdump) @@ -131,6 +131,7 @@ CTL_PROTO(arenas_nbins) CTL_PROTO(arenas_nhbins) CTL_PROTO(arenas_nlruns) CTL_PROTO(arenas_extend) +CTL_PROTO(prof_thread_active_init) CTL_PROTO(prof_active) CTL_PROTO(prof_dump) CTL_PROTO(prof_reset) @@ -253,6 +254,7 @@ static const ctl_named_node_t opt_node[] = { {NAME("prof"), CTL(opt_prof)}, {NAME("prof_prefix"), CTL(opt_prof_prefix)}, {NAME("prof_active"), CTL(opt_prof_active)}, + {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, {NAME("prof_gdump"), CTL(opt_prof_gdump)}, @@ -318,6 +320,7 @@ static const ctl_named_node_t arenas_node[] = { }; static const ctl_named_node_t prof_node[] = { + {NAME("thread_active_init"), CTL(prof_thread_active_init)}, {NAME("active"), CTL(prof_active)}, {NAME("dump"), CTL(prof_dump)}, {NAME("reset"), CTL(prof_reset)}, @@ -979,6 +982,14 @@ ctl_postfork_child(void) } \ } while (0) +#define READ_XOR_WRITE() do { \ + if ((oldp != NULL && oldlenp != NULL) && (newp != NULL || \ + newlen != 0)) { \ + ret = EPERM; \ + goto label_return; \ + } \ +} while (0) + #define READ(v, t) do { \ if (oldp != NULL && oldlenp != NULL) { \ if (*oldlenp != sizeof(t)) { \ @@ -1208,7 +1219,9 @@ CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) -CTL_RO_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) /* Mutable. */ +CTL_RO_NL_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_thread_active_init, + opt_prof_thread_active_init, bool) CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) @@ -1332,12 +1345,12 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - const char *oldname; if (!config_prof) return (ENOENT); - oldname = prof_thread_name_get(); + READ_XOR_WRITE(); + if (newp != NULL) { tsd_t *tsd; @@ -1352,12 +1365,13 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp, goto label_return; } - if (prof_thread_name_set(tsd, *(const char **)newp)) { - ret = EAGAIN; + if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != + 0) goto label_return; - } + } else { + const char *oldname = prof_thread_name_get(); + READ(oldname, const char *); } - READ(oldname, const char *); ret = 0; label_return: @@ -1660,6 +1674,31 @@ label_return: /******************************************************************************/ +static int +prof_thread_active_init_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + bool oldval; + + if (!config_prof) + return (ENOENT); + + if (newp != NULL) { + if (newlen != sizeof(bool)) { + ret = EINVAL; + goto label_return; + } + oldval = prof_thread_active_init_set(*(bool *)newp); + } else + oldval = prof_thread_active_init_get(); + READ(oldval, bool); + + ret = 0; +label_return: + return (ret); +} + static int prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) @@ -1670,22 +1709,18 @@ prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, if (!config_prof) return (ENOENT); - malloc_mutex_lock(&ctl_mtx); /* Protect opt_prof_active. */ - oldval = opt_prof_active; if (newp != NULL) { - /* - * The memory barriers will tend to make opt_prof_active - * propagate faster on systems with weak memory ordering. - */ - mb_write(); - WRITE(opt_prof_active, bool); - mb_write(); - } + if (newlen != sizeof(bool)) { + ret = EINVAL; + goto label_return; + } + oldval = prof_active_set(*(bool *)newp); + } else + oldval = prof_active_get(); READ(oldval, bool); ret = 0; label_return: - malloc_mutex_unlock(&ctl_mtx); return (ret); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 0d041313..2e96705d 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -655,6 +655,8 @@ malloc_conf_init(void) "prof_prefix", "jeprof") CONF_HANDLE_BOOL(opt_prof_active, "prof_active", true) + CONF_HANDLE_BOOL(opt_prof_thread_active_init, + "prof_thread_active_init", true) CONF_HANDLE_SIZE_T(opt_lg_prof_sample, "lg_prof_sample", 0, (sizeof(uint64_t) << 3) - 1, true) diff --git a/src/prof.c b/src/prof.c index 29b4baaa..5b979989 100644 --- a/src/prof.c +++ b/src/prof.c @@ -16,6 +16,7 @@ bool opt_prof = false; bool opt_prof_active = true; +bool opt_prof_thread_active_init = true; size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; bool opt_prof_gdump = false; @@ -29,6 +30,20 @@ char opt_prof_prefix[ #endif 1]; +/* + * Initialized as opt_prof_active, and accessed via + * prof_active_[gs]et{_unlocked,}(). + */ +bool prof_active; +static malloc_mutex_t prof_active_mtx; + +/* + * Initialized as opt_prof_thread_active_init, and accessed via + * prof_thread_active_init_[gs]et(). + */ +static bool prof_thread_active_init; +static malloc_mutex_t prof_thread_active_init_mtx; + uint64_t prof_interval = 0; size_t lg_prof_sample; @@ -103,6 +118,7 @@ static bool prof_tctx_should_destroy(prof_tctx_t *tctx); static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); static bool prof_tdata_should_destroy(prof_tdata_t *tdata); static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata); +static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name); /******************************************************************************/ /* Red-black trees. */ @@ -1593,7 +1609,8 @@ prof_thr_uid_alloc(void) } static prof_tdata_t * -prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim) +prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, + char *thread_name, bool active) { prof_tdata_t *tdata; @@ -1607,7 +1624,7 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim) tdata->lock = prof_tdata_mutex_choose(thr_uid); tdata->thr_uid = thr_uid; tdata->thr_discrim = thr_discrim; - tdata->thread_name = NULL; + tdata->thread_name = thread_name; tdata->attached = true; tdata->expired = false; @@ -1625,7 +1642,7 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim) tdata->enq_gdump = false; tdata->dumping = false; - tdata->active = true; + tdata->active = active; malloc_mutex_lock(&tdatas_mtx); tdata_tree_insert(&tdatas, tdata); @@ -1638,7 +1655,8 @@ prof_tdata_t * prof_tdata_init(tsd_t *tsd) { - return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0)); + return (prof_tdata_init_impl(tsd, prof_thr_uid_alloc(), 0, NULL, + prof_thread_active_init_get())); } /* tdata->lock must be held. */ @@ -1698,9 +1716,13 @@ prof_tdata_reinit(tsd_t *tsd, prof_tdata_t *tdata) { uint64_t thr_uid = tdata->thr_uid; uint64_t thr_discrim = tdata->thr_discrim + 1; + char *thread_name = (tdata->thread_name != NULL) ? + prof_thread_name_alloc(tsd, tdata->thread_name) : NULL; + bool active = tdata->active; prof_tdata_detach(tsd, tdata); - return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim)); + return (prof_tdata_init_impl(tsd, thr_uid, thr_discrim, thread_name, + active)); } static bool @@ -1768,6 +1790,29 @@ prof_tdata_cleanup(tsd_t *tsd) prof_tdata_detach(tsd, tdata); } +bool +prof_active_get(void) +{ + bool prof_active_current; + + malloc_mutex_lock(&prof_active_mtx); + prof_active_current = prof_active; + malloc_mutex_unlock(&prof_active_mtx); + return (prof_active_current); +} + +bool +prof_active_set(bool active) +{ + bool prof_active_old; + + malloc_mutex_lock(&prof_active_mtx); + prof_active_old = prof_active; + prof_active = active; + malloc_mutex_unlock(&prof_active_mtx); + return (prof_active_old); +} + const char * prof_thread_name_get(void) { @@ -1775,34 +1820,64 @@ prof_thread_name_get(void) prof_tdata_t *tdata; if ((tsd = tsd_tryget()) == NULL) - return (NULL); + return (""); tdata = prof_tdata_get(tsd, true); if (tdata == NULL) - return (NULL); - return (tdata->thread_name); + return (""); + return (tdata->thread_name != NULL ? tdata->thread_name : ""); } -bool +static char * +prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) +{ + char *ret; + size_t size; + + if (thread_name == NULL) + return (NULL); + + size = strlen(thread_name) + 1; + if (size == 1) + return (""); + + ret = imalloc(tsd, size); + if (ret == NULL) + return (NULL); + memcpy(ret, thread_name, size); + return (ret); +} + +int prof_thread_name_set(tsd_t *tsd, const char *thread_name) { prof_tdata_t *tdata; - size_t size; + unsigned i; char *s; tdata = prof_tdata_get(tsd, true); if (tdata == NULL) - return (true); + return (EAGAIN); - size = strlen(thread_name) + 1; - s = imalloc(tsd, size); + /* Validate input. */ + if (thread_name == NULL) + return (EFAULT); + for (i = 0; thread_name[i] != '\0'; i++) { + char c = thread_name[i]; + if (!isgraph(c) && !isblank(c)) + return (EFAULT); + } + + s = prof_thread_name_alloc(tsd, thread_name); if (s == NULL) - return (true); + return (EAGAIN); - memcpy(s, thread_name, size); - if (tdata->thread_name != NULL) + if (tdata->thread_name != NULL) { idalloc(tsd, tdata->thread_name); - tdata->thread_name = s; - return (false); + tdata->thread_name = NULL; + } + if (strlen(s) > 0) + tdata->thread_name = s; + return (0); } bool @@ -1834,6 +1909,29 @@ prof_thread_active_set(bool active) return (false); } +bool +prof_thread_active_init_get(void) +{ + bool active_init; + + malloc_mutex_lock(&prof_thread_active_init_mtx); + active_init = prof_thread_active_init; + malloc_mutex_unlock(&prof_thread_active_init_mtx); + return (active_init); +} + +bool +prof_thread_active_init_set(bool active_init) +{ + bool active_init_old; + + malloc_mutex_lock(&prof_thread_active_init_mtx); + active_init_old = prof_thread_active_init; + prof_thread_active_init = active_init; + malloc_mutex_unlock(&prof_thread_active_init_mtx); + return (active_init_old); +} + void prof_boot0(void) { @@ -1882,6 +1980,14 @@ prof_boot2(void) lg_prof_sample = opt_lg_prof_sample; + prof_active = opt_prof_active; + if (malloc_mutex_init(&prof_active_mtx)) + return (true); + + prof_thread_active_init = opt_prof_thread_active_init; + if (malloc_mutex_init(&prof_thread_active_init_mtx)) + return (true); + if ((tsd = tsd_tryget()) == NULL) return (true); if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, diff --git a/src/stats.c b/src/stats.c index aa095500..5c3d7017 100644 --- a/src/stats.c +++ b/src/stats.c @@ -336,7 +336,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "___ Begin jemalloc statistics ___\n"); if (general) { - int err; const char *cpv; bool bv; unsigned uv; @@ -355,26 +354,31 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, bv ? "enabled" : "disabled"); #define OPT_WRITE_BOOL(n) \ - if ((err = je_mallctl("opt."#n, &bv, &bsz, NULL, 0)) \ - == 0) { \ + if (je_mallctl("opt."#n, &bv, &bsz, NULL, 0) == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %s\n", bv ? "true" : "false"); \ } +#define OPT_WRITE_BOOL_MUTABLE(n, m) { \ + bool bv2; \ + if (je_mallctl("opt."#n, &bv, &bsz, NULL, 0) == 0 && \ + je_mallctl(#m, &bv2, &bsz, NULL, 0) == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %s ("#m": %s)\n", bv ? "true" \ + : "false", bv2 ? "true" : "false"); \ + } \ +} #define OPT_WRITE_SIZE_T(n) \ - if ((err = je_mallctl("opt."#n, &sv, &ssz, NULL, 0)) \ - == 0) { \ + if (je_mallctl("opt."#n, &sv, &ssz, NULL, 0) == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %zu\n", sv); \ } #define OPT_WRITE_SSIZE_T(n) \ - if ((err = je_mallctl("opt."#n, &ssv, &sssz, NULL, 0)) \ - == 0) { \ + if (je_mallctl("opt."#n, &ssv, &sssz, NULL, 0) == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %zd\n", ssv); \ } #define OPT_WRITE_CHAR_P(n) \ - if ((err = je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0)) \ - == 0) { \ + if (je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0) == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": \"%s\"\n", cpv); \ } @@ -398,7 +402,9 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, OPT_WRITE_SSIZE_T(lg_tcache_max) OPT_WRITE_BOOL(prof) OPT_WRITE_CHAR_P(prof_prefix) - OPT_WRITE_BOOL(prof_active) + OPT_WRITE_BOOL_MUTABLE(prof_active, prof.active) + OPT_WRITE_BOOL_MUTABLE(prof_thread_active_init, + prof.thread_active_init) OPT_WRITE_SSIZE_T(lg_prof_sample) OPT_WRITE_BOOL(prof_accum) OPT_WRITE_SSIZE_T(lg_prof_interval) @@ -407,6 +413,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, OPT_WRITE_BOOL(prof_leak) #undef OPT_WRITE_BOOL +#undef OPT_WRITE_BOOL_MUTABLE #undef OPT_WRITE_SIZE_T #undef OPT_WRITE_SSIZE_T #undef OPT_WRITE_CHAR_P @@ -434,13 +441,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "Min active:dirty page ratio per arena: N/A\n"); } - if ((err = je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0)) - == 0) { + if (je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0) == 0) { malloc_cprintf(write_cb, cbopaque, "Maximum thread-cached size class: %zu\n", sv); } - if ((err = je_mallctl("opt.prof", &bv, &bsz, NULL, 0)) == 0 && - bv) { + if (je_mallctl("opt.prof", &bv, &bsz, NULL, 0) == 0 && bv) { CTL_GET("prof.lg_sample", &sv, size_t); malloc_cprintf(write_cb, cbopaque, "Average profile sample interval: %"PRIu64 diff --git a/test/unit/prof_active.c b/test/unit/prof_active.c new file mode 100644 index 00000000..d4bab8d0 --- /dev/null +++ b/test/unit/prof_active.c @@ -0,0 +1,136 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = + "prof:true,prof_thread_active_init:false,lg_prof_sample:0,prof_final:false"; +#endif + +static void +mallctl_bool_get(const char *name, bool expected, const char *func, int line) +{ + bool old; + size_t sz; + + sz = sizeof(old); + assert_d_eq(mallctl(name, &old, &sz, NULL, 0), 0, + "%s():%d: Unexpected mallctl failure reading %s", func, line, name); + assert_b_eq(old, expected, "%s():%d: Unexpected %s value", func, line, + name); +} + +static void +mallctl_bool_set(const char *name, bool old_expected, bool val_new, + const char *func, int line) +{ + bool old; + size_t sz; + + sz = sizeof(old); + assert_d_eq(mallctl(name, &old, &sz, &val_new, sizeof(val_new)), 0, + "%s():%d: Unexpected mallctl failure reading/writing %s", func, + line, name); + assert_b_eq(old, old_expected, "%s():%d: Unexpected %s value", func, + line, name); +} + +static void +mallctl_prof_active_get_impl(bool prof_active_old_expected, const char *func, + int line) +{ + + mallctl_bool_get("prof.active", prof_active_old_expected, func, line); +} +#define mallctl_prof_active_get(a) \ + mallctl_prof_active_get_impl(a, __func__, __LINE__) + +static void +mallctl_prof_active_set_impl(bool prof_active_old_expected, + bool prof_active_new, const char *func, int line) +{ + + mallctl_bool_set("prof.active", prof_active_old_expected, + prof_active_new, func, line); +} +#define mallctl_prof_active_set(a, b) \ + mallctl_prof_active_set_impl(a, b, __func__, __LINE__) + +static void +mallctl_thread_prof_active_get_impl(bool thread_prof_active_old_expected, + const char *func, int line) +{ + + mallctl_bool_get("thread.prof.active", thread_prof_active_old_expected, + func, line); +} +#define mallctl_thread_prof_active_get(a) \ + mallctl_thread_prof_active_get_impl(a, __func__, __LINE__) + +static void +mallctl_thread_prof_active_set_impl(bool thread_prof_active_old_expected, + bool thread_prof_active_new, const char *func, int line) +{ + + mallctl_bool_set("thread.prof.active", thread_prof_active_old_expected, + thread_prof_active_new, func, line); +} +#define mallctl_thread_prof_active_set(a, b) \ + mallctl_thread_prof_active_set_impl(a, b, __func__, __LINE__) + +static void +prof_sampling_probe_impl(bool expect_sample, const char *func, int line) +{ + void *p; + size_t expected_backtraces = expect_sample ? 1 : 0; + + assert_zu_eq(prof_bt_count(), 0, "%s():%d: Expected 0 backtraces", func, + line); + p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + assert_zu_eq(prof_bt_count(), expected_backtraces, + "%s():%d: Unexpected backtrace count", func, line); + dallocx(p, 0); +} +#define prof_sampling_probe(a) \ + prof_sampling_probe_impl(a, __func__, __LINE__) + +TEST_BEGIN(test_prof_active) +{ + + test_skip_if(!config_prof); + + mallctl_prof_active_get(true); + mallctl_thread_prof_active_get(false); + + mallctl_prof_active_set(true, true); + mallctl_thread_prof_active_set(false, false); + /* prof.active, !thread.prof.active. */ + prof_sampling_probe(false); + + mallctl_prof_active_set(true, false); + mallctl_thread_prof_active_set(false, false); + /* !prof.active, !thread.prof.active. */ + prof_sampling_probe(false); + + mallctl_prof_active_set(false, false); + mallctl_thread_prof_active_set(false, true); + /* !prof.active, thread.prof.active. */ + prof_sampling_probe(false); + + mallctl_prof_active_set(false, true); + mallctl_thread_prof_active_set(true, true); + /* prof.active, thread.prof.active. */ + prof_sampling_probe(true); + + /* Restore settings. */ + mallctl_prof_active_set(true, true); + mallctl_thread_prof_active_set(true, false); +} +TEST_END + +int +main(void) +{ + + return (test( + test_prof_active)); +} diff --git a/test/unit/prof_reset.c b/test/unit/prof_reset.c index 62a4d5af..3af19642 100644 --- a/test/unit/prof_reset.c +++ b/test/unit/prof_reset.c @@ -22,6 +22,8 @@ TEST_BEGIN(test_prof_reset_basic) size_t sz; unsigned i; + test_skip_if(!config_prof); + sz = sizeof(size_t); assert_d_eq(mallctl("opt.lg_prof_sample", &lg_prof_sample_orig, &sz, NULL, 0), 0, @@ -90,6 +92,8 @@ TEST_BEGIN(test_prof_reset_cleanup) void *p; prof_dump_header_t *prof_dump_header_orig; + test_skip_if(!config_prof); + active = true; assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), 0, "Unexpected mallctl failure while activating profiling"); diff --git a/test/unit/prof_thread_name.c b/test/unit/prof_thread_name.c new file mode 100644 index 00000000..7fb8038a --- /dev/null +++ b/test/unit/prof_thread_name.c @@ -0,0 +1,128 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = + "prof:true,prof_active:false,prof_final:false"; +#endif + +static void +mallctl_thread_name_get_impl(const char *thread_name_expected, const char *func, + int line) +{ + const char *thread_name_old; + size_t sz; + + sz = sizeof(thread_name_old); + assert_d_eq(mallctl("thread.prof.name", &thread_name_old, &sz, NULL, 0), + 0, "%s():%d: Unexpected mallctl failure reading thread.prof.name", + func, line); + assert_str_eq(thread_name_old, thread_name_expected, + "%s():%d: Unexpected thread.prof.name value", func, line); +} +#define mallctl_thread_name_get(a) \ + mallctl_thread_name_get_impl(a, __func__, __LINE__) + +static void +mallctl_thread_name_set_impl(const char *thread_name, const char *func, + int line) +{ + + assert_d_eq(mallctl("thread.prof.name", NULL, NULL, &thread_name, + sizeof(thread_name)), 0, + "%s():%d: Unexpected mallctl failure reading thread.prof.name", + func, line); + mallctl_thread_name_get_impl(thread_name, func, line); +} +#define mallctl_thread_name_set(a) \ + mallctl_thread_name_set_impl(a, __func__, __LINE__) + +TEST_BEGIN(test_prof_thread_name_validation) +{ + const char *thread_name; + + mallctl_thread_name_get(""); + mallctl_thread_name_set("hi there"); + + /* NULL input shouldn't be allowed. */ + thread_name = NULL; + assert_d_eq(mallctl("thread.prof.name", NULL, NULL, &thread_name, + sizeof(thread_name)), EFAULT, + "Unexpected mallctl result writing \"%s\" to thread.prof.name", + thread_name); + + /* '\n' shouldn't be allowed. */ + thread_name = "hi\nthere"; + assert_d_eq(mallctl("thread.prof.name", NULL, NULL, &thread_name, + sizeof(thread_name)), EFAULT, + "Unexpected mallctl result writing \"%s\" to thread.prof.name", + thread_name); + + /* Simultaneous read/write shouldn't be allowed. */ + { + const char *thread_name_old; + size_t sz; + + sz = sizeof(thread_name_old); + assert_d_eq(mallctl("thread.prof.name", &thread_name_old, &sz, + &thread_name, sizeof(thread_name)), EPERM, + "Unexpected mallctl result writing \"%s\" to " + "thread.prof.name", thread_name); + } + + mallctl_thread_name_set(""); +} +TEST_END + +#define NTHREADS 4 +#define NRESET 25 +static void * +thd_start(void *varg) +{ + unsigned thd_ind = *(unsigned *)varg; + char thread_name[16] = ""; + unsigned i; + + malloc_snprintf(thread_name, sizeof(thread_name), "thread %u", thd_ind); + + mallctl_thread_name_get(""); + mallctl_thread_name_set(thread_name); + + for (i = 0; i < NRESET; i++) { + assert_d_eq(mallctl("prof.reset", NULL, NULL, NULL, 0), 0, + "Unexpected error while resetting heap profile data"); + mallctl_thread_name_get(thread_name); + } + + mallctl_thread_name_set(thread_name); + mallctl_thread_name_set(""); + + return (NULL); +} + +TEST_BEGIN(test_prof_thread_name_threaded) +{ + thd_t thds[NTHREADS]; + unsigned thd_args[NTHREADS]; + unsigned i; + + test_skip_if(!config_prof); + + for (i = 0; i < NTHREADS; i++) { + thd_args[i] = i; + thd_create(&thds[i], thd_start, (void *)&thd_args[i]); + } + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); +} +TEST_END +#undef NTHREADS +#undef NRESET + +int +main(void) +{ + + return (test( + test_prof_thread_name_validation, + test_prof_thread_name_threaded)); +} From b72d4abc5fb1185e4017c014d521693a99f9175b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 3 Oct 2014 23:41:53 -0700 Subject: [PATCH 127/352] Skip test_prof_thread_name_validation if !config_prof. --- test/unit/prof_thread_name.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/prof_thread_name.c b/test/unit/prof_thread_name.c index 7fb8038a..6066dba7 100644 --- a/test/unit/prof_thread_name.c +++ b/test/unit/prof_thread_name.c @@ -40,6 +40,8 @@ TEST_BEGIN(test_prof_thread_name_validation) { const char *thread_name; + test_skip_if(!config_prof); + mallctl_thread_name_get(""); mallctl_thread_name_set("hi there"); From a4a972d9a163a57183f851535104f4e8ac78f511 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 00:35:07 -0700 Subject: [PATCH 128/352] Fix install_lib target (incorrect jemalloc.pc path). --- Makefile.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.in b/Makefile.in index 52f5a9d2..50f6596a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -101,7 +101,7 @@ DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV) ifneq ($(SOREV),$(SO)) DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO) endif -PC := $(srcroot)jemalloc.pc +PC := $(objroot)jemalloc.pc MAN3 := $(objroot)doc/jemalloc$(install_suffix).3 DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) From 029d44cf8b22aa7b749747bfd585887fb59e0030 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 11:12:53 -0700 Subject: [PATCH 129/352] Fix tsd cleanup regressions. Fix tsd cleanup regressions that were introduced in 5460aa6f6676c7f253bfcb75c028dfd38cae8aaf (Convert all tsd variables to reside in a single tsd structure.). These regressions were twofold: 1) tsd_tryget() should never (and need never) return NULL. Rename it to tsd_fetch() and simplify all callers. 2) tsd_*_set() must only be called when tsd is in the nominal state, because cleanup happens during the nominal-->purgatory transition, and re-initialization must not happen while in the purgatory state. Add tsd_nominal() and use it as needed. Note that tsd_*{p,}_get() can still be used as long as no re-initialization that would require cleanup occurs. This means that e.g. the thread_allocated counter can be updated unconditionally. --- include/jemalloc/internal/private_symbols.txt | 3 +- include/jemalloc/internal/prof.h | 6 +- include/jemalloc/internal/quarantine.h | 4 +- include/jemalloc/internal/tcache.h | 21 ++--- include/jemalloc/internal/tsd.h | 65 +++++++++------- src/ctl.c | 26 ++----- src/jemalloc.c | 78 +++++++++---------- src/prof.c | 29 +++---- src/tcache.c | 3 +- src/tsd.c | 5 -- test/unit/ckh.c | 9 +-- test/unit/tsd.c | 35 ++++++++- 12 files changed, 137 insertions(+), 147 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 63657833..4ea9a953 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -390,12 +390,14 @@ tsd_arena_set tsd_boot tsd_cleanup tsd_cleanup_wrapper +tsd_fetch tsd_get tsd_get_wrapper tsd_initialized tsd_init_check_recursion tsd_init_finish tsd_init_head +tsd_nominal tsd_quarantine_get tsd_quarantine_set tsd_set @@ -411,7 +413,6 @@ tsd_thread_allocated_get tsd_thread_allocated_set tsd_thread_deallocated_get tsd_thread_deallocated_set -tsd_tryget u2rz valgrind_freelike_block valgrind_make_mem_defined diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 3d3f8f4e..0ec7c18a 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -331,8 +331,10 @@ prof_tdata_get(tsd_t *tsd, bool create) tdata = tsd_prof_tdata_get(tsd); if (create) { if (unlikely(tdata == NULL)) { - tdata = prof_tdata_init(tsd); - tsd_prof_tdata_set(tsd, tdata); + if (tsd_nominal(tsd)) { + tdata = prof_tdata_init(tsd); + tsd_prof_tdata_set(tsd, tdata); + } } else if (unlikely(tdata->expired)) { tdata = prof_tdata_reinit(tsd, tdata); tsd_prof_tdata_set(tsd, tdata); diff --git a/include/jemalloc/internal/quarantine.h b/include/jemalloc/internal/quarantine.h index 3a755985..4e9c710a 100644 --- a/include/jemalloc/internal/quarantine.h +++ b/include/jemalloc/internal/quarantine.h @@ -49,8 +49,8 @@ quarantine_alloc_hook(void) assert(config_fill && opt_quarantine); - tsd = tsd_tryget(); - if (tsd != NULL && tsd_quarantine_get(tsd) == NULL) + tsd = tsd_fetch(); + if (tsd_quarantine_get(tsd) == NULL && tsd_nominal(tsd)) tsd_quarantine_set(tsd, quarantine_init(tsd, LG_MAXOBJS_INIT)); } #endif diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index bc0b41c7..1a70972c 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -142,9 +142,8 @@ tcache_flush(void) cassert(config_tcache); - tsd = tsd_tryget(); - if (tsd != NULL) - tcache_cleanup(tsd); + tsd = tsd_fetch(); + tcache_cleanup(tsd); } JEMALLOC_INLINE bool @@ -155,9 +154,7 @@ tcache_enabled_get(void) cassert(config_tcache); - tsd = tsd_tryget(); - if (tsd == NULL) - return (false); + tsd = tsd_fetch(); tcache_enabled = tsd_tcache_enabled_get(tsd); if (tcache_enabled == tcache_enabled_default) { tcache_enabled = (tcache_enabled_t)opt_tcache; @@ -175,9 +172,7 @@ tcache_enabled_set(bool enabled) cassert(config_tcache); - tsd = tsd_tryget(); - if (tsd == NULL) - return; + tsd = tsd_fetch(); tcache_enabled = (tcache_enabled_t)enabled; tsd_tcache_enabled_set(tsd, tcache_enabled); @@ -195,17 +190,11 @@ tcache_get(tsd_t *tsd, bool create) return (NULL); if (config_lazy_lock && !isthreaded) return (NULL); - /* - * If create is true, the caller has already assured that tsd is - * non-NULL. - */ - if (!create && unlikely(tsd == NULL)) - return (NULL); tcache = tsd_tcache_get(tsd); if (!create) return (tcache); - if (unlikely(tcache == NULL)) { + if (unlikely(tcache == NULL) && tsd_nominal(tsd)) { tcache = tcache_get_hard(tsd); tsd_tcache_set(tsd, tcache); } diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 44952eed..25450391 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -49,16 +49,19 @@ typedef enum { * Note that all of the functions deal in terms of (a_type *) rather than * (a_type) so that it is possible to support non-pointer types (unlike * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is - * cast to (void *). This means that the cleanup function needs to cast *and* - * dereference the function argument, e.g.: + * cast to (void *). This means that the cleanup function needs to cast the + * function argument to (a_type *), then dereference the resulting pointer to + * access fields, e.g. * - * bool + * void * example_tsd_cleanup(void *arg) * { - * example_t *example = *(example_t **)arg; + * example_t *example = (example_t *)arg; * + * example->x = 42; * [...] - * return ([want the cleanup function to be called again]); + * if ([want the cleanup function to be called again]) + * example_tsd_set(example); * } * * If example_tsd_set() is called within example_tsd_cleanup(), it will be @@ -468,7 +471,8 @@ void tsd_cleanup(void *arg); #ifndef JEMALLOC_ENABLE_INLINE malloc_tsd_protos(JEMALLOC_ATTR(unused), , tsd_t) -tsd_t *tsd_tryget(void); +tsd_t *tsd_fetch(void); +bool tsd_nominal(tsd_t *tsd); #define O(n, t) \ t *tsd_##n##p_get(tsd_t *tsd); \ t tsd_##n##_get(tsd_t *tsd); \ @@ -481,50 +485,53 @@ MALLOC_TSD malloc_tsd_externs(, tsd_t) malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, , tsd_t, tsd_initializer, tsd_cleanup) -JEMALLOC_INLINE tsd_t * -tsd_tryget(void) +JEMALLOC_ALWAYS_INLINE tsd_t * +tsd_fetch(void) { - tsd_t *tsd; + tsd_t *tsd = tsd_get(); - tsd = tsd_get(); - if (unlikely(tsd == NULL)) - return (NULL); - - if (likely(tsd->state == tsd_state_nominal)) - return (tsd); - else if (tsd->state == tsd_state_uninitialized) { - tsd->state = tsd_state_nominal; - tsd_set(tsd); - return (tsd); - } else if (tsd->state == tsd_state_purgatory) { - tsd->state = tsd_state_reincarnated; - tsd_set(tsd); - return (NULL); - } else { - assert(tsd->state == tsd_state_reincarnated); - return (NULL); + if (unlikely(tsd->state != tsd_state_nominal)) { + if (tsd->state == tsd_state_uninitialized) { + tsd->state = tsd_state_nominal; + /* Trigger cleanup handler registration. */ + tsd_set(tsd); + } else if (tsd->state == tsd_state_purgatory) { + tsd->state = tsd_state_reincarnated; + tsd_set(tsd); + } else + assert(tsd->state == tsd_state_reincarnated); } + + return (tsd); +} + +JEMALLOC_INLINE bool +tsd_nominal(tsd_t *tsd) +{ + + return (tsd->state == tsd_state_nominal); } #define O(n, t) \ -JEMALLOC_INLINE t * \ +JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get(tsd_t *tsd) \ { \ \ return (&tsd->n); \ } \ \ -JEMALLOC_INLINE t \ +JEMALLOC_ALWAYS_INLINE t \ tsd_##n##_get(tsd_t *tsd) \ { \ \ return (*tsd_##n##p_get(tsd)); \ } \ \ -JEMALLOC_INLINE void \ +JEMALLOC_ALWAYS_INLINE void \ tsd_##n##_set(tsd_t *tsd, t n) \ { \ \ + assert(tsd->state == tsd_state_nominal); \ tsd->n = n; \ } MALLOC_TSD diff --git a/src/ctl.c b/src/ctl.c index 8f9faa56..309f1f65 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -571,9 +571,7 @@ ctl_grow(void) ctl_arena_stats_t *astats; arena_t **tarenas; - tsd = tsd_tryget(); - if (tsd == NULL) - return (true); + tsd = tsd_fetch(); /* Allocate extended arena stats and arenas arrays. */ astats = (ctl_arena_stats_t *)imalloc(tsd, (ctl_stats.narenas + 2) * @@ -1132,11 +1130,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ if (!(c)) \ return (ENOENT); \ READONLY(); \ - tsd = tsd_tryget(); \ - if (tsd == NULL) { \ - ret = EAGAIN; \ - goto label_return; \ - } \ + tsd = tsd_fetch(); \ oldval = (m(tsd)); \ READ(oldval, t); \ \ @@ -1239,9 +1233,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, tsd_t *tsd; unsigned newind, oldind; - tsd = tsd_tryget(); - if (tsd == NULL) - return (EAGAIN); + tsd = tsd_fetch(); malloc_mutex_lock(&ctl_mtx); newind = oldind = choose_arena(tsd, NULL)->ind; @@ -1359,11 +1351,7 @@ thread_prof_name_ctl(const size_t *mib, size_t miblen, void *oldp, goto label_return; } - tsd = tsd_tryget(); - if (tsd == NULL) { - ret = EAGAIN; - goto label_return; - } + tsd = tsd_fetch(); if ((ret = prof_thread_name_set(tsd, *(const char **)newp)) != 0) @@ -1763,11 +1751,7 @@ prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, if (lg_sample >= (sizeof(uint64_t) << 3)) lg_sample = (sizeof(uint64_t) << 3) - 1; - tsd = tsd_tryget(); - if (tsd == NULL) { - ret = EAGAIN; - goto label_return; - } + tsd = tsd_fetch(); prof_reset(tsd, lg_sample); diff --git a/src/jemalloc.c b/src/jemalloc.c index 2e96705d..4a3d968b 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -194,7 +194,8 @@ choose_arena_hard(tsd_t *tsd) malloc_mutex_unlock(&arenas_lock); } - tsd_arena_set(tsd, ret); + if (tsd_nominal(tsd)) + tsd_arena_set(tsd, ret); return (ret); } @@ -908,8 +909,9 @@ JEMALLOC_ALWAYS_INLINE_C void * imalloc_body(size_t size, tsd_t **tsd, size_t *usize) { - if (unlikely(malloc_init()) || unlikely((*tsd = tsd_tryget()) == NULL)) + if (unlikely(malloc_init())) return (NULL); + *tsd = tsd_fetch(); if (config_prof && opt_prof) { *usize = s2u(size); @@ -1000,10 +1002,11 @@ imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) assert(min_alignment != 0); - if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) { + if (unlikely(malloc_init())) { result = NULL; goto label_oom; } else { + tsd = tsd_fetch(); if (size == 0) size = 1; @@ -1124,11 +1127,12 @@ je_calloc(size_t num, size_t size) size_t num_size; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) { + if (unlikely(malloc_init())) { num_size = 0; ret = NULL; goto label_return; } + tsd = tsd_fetch(); num_size = num * size; if (unlikely(num_size == 0)) { @@ -1228,7 +1232,7 @@ ifree(tsd_t *tsd, void *ptr, bool try_tcache) prof_free(tsd, ptr, usize); } else if (config_stats || config_valgrind) usize = isalloc(ptr, config_prof); - if (config_stats && likely(tsd != NULL)) + if (config_stats) *tsd_thread_deallocatedp_get(tsd) += usize; if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); @@ -1246,7 +1250,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache) if (config_prof && opt_prof) prof_free(tsd, ptr, usize); - if (config_stats && likely(tsd != NULL)) + if (config_stats) *tsd_thread_deallocatedp_get(tsd) += usize; if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); @@ -1267,7 +1271,7 @@ je_realloc(void *ptr, size_t size) if (ptr != NULL) { /* realloc(ptr, 0) is equivalent to free(ptr). */ UTRACE(ptr, 0, 0); - tsd = tsd_tryget(); + tsd = tsd_fetch(); ifree(tsd, ptr, true); return (NULL); } @@ -1277,27 +1281,23 @@ je_realloc(void *ptr, size_t size) if (likely(ptr != NULL)) { assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); + tsd = tsd_fetch(); - if ((tsd = tsd_tryget()) != NULL) { - if ((config_prof && opt_prof) || config_stats || - (config_valgrind && unlikely(in_valgrind))) - old_usize = isalloc(ptr, config_prof); - if (config_valgrind && unlikely(in_valgrind)) { - old_rzsize = config_prof ? p2rz(ptr) : - u2rz(old_usize); - } + if ((config_prof && opt_prof) || config_stats || + (config_valgrind && unlikely(in_valgrind))) + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && unlikely(in_valgrind)) + old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); - if (config_prof && opt_prof) { + if (config_prof && opt_prof) { + usize = s2u(size); + ret = irealloc_prof(tsd, ptr, old_usize, usize); + } else { + if (config_stats || (config_valgrind && + unlikely(in_valgrind))) usize = s2u(size); - ret = irealloc_prof(tsd, ptr, old_usize, usize); - } else { - if (config_stats || (config_valgrind && - unlikely(in_valgrind))) - usize = s2u(size); - ret = iralloc(tsd, ptr, size, 0, false); - } - } else - ret = NULL; + ret = iralloc(tsd, ptr, size, 0, false); + } } else { /* realloc(NULL, size) is equivalent to malloc(size). */ ret = imalloc_body(size, &tsd, &usize); @@ -1313,10 +1313,8 @@ je_realloc(void *ptr, size_t size) } if (config_stats && likely(ret != NULL)) { assert(usize == isalloc(ret, config_prof)); - if (tsd != NULL) { - *tsd_thread_allocatedp_get(tsd) += usize; - *tsd_thread_deallocatedp_get(tsd) += old_usize; - } + *tsd_thread_allocatedp_get(tsd) += usize; + *tsd_thread_deallocatedp_get(tsd) += old_usize; } UTRACE(ptr, size, ret); JEMALLOC_VALGRIND_REALLOC(true, ret, usize, true, ptr, old_usize, @@ -1330,7 +1328,7 @@ je_free(void *ptr) UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) - ifree(tsd_tryget(), ptr, true); + ifree(tsd_fetch(), ptr, true); } /* @@ -1543,8 +1541,9 @@ je_mallocx(size_t size, int flags) assert(size != 0); - if (unlikely(malloc_init()) || unlikely((tsd = tsd_tryget()) == NULL)) + if (unlikely(malloc_init())) goto label_oom; + tsd = tsd_fetch(); if (config_prof && opt_prof) p = imallocx_prof(tsd, size, flags, &usize); @@ -1554,10 +1553,8 @@ je_mallocx(size_t size, int flags) goto label_oom; if (config_stats) { - tsd_t *tsd = tsd_tryget(); assert(usize == isalloc(p, config_prof)); - if (tsd != NULL) - *tsd_thread_allocatedp_get(tsd) += usize; + *tsd_thread_allocatedp_get(tsd) += usize; } UTRACE(0, size, p); JEMALLOC_VALGRIND_MALLOC(true, p, usize, MALLOCX_ZERO_GET(flags)); @@ -1649,9 +1646,7 @@ je_rallocx(void *ptr, size_t size, int flags) assert(size != 0); assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); - - if (unlikely((tsd = tsd_tryget()) == NULL)) - goto label_oom; + tsd = tsd_fetch(); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); @@ -1794,6 +1789,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) assert(SIZE_T_MAX - size >= extra); assert(malloc_initialized || IS_INITIALIZER); malloc_thread_init(); + tsd = tsd_fetch(); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); @@ -1802,10 +1798,6 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) arena = NULL; old_usize = isalloc(ptr, config_prof); - if (unlikely((tsd = tsd_tryget()) == NULL)) { - usize = old_usize; - goto label_not_resized; - } if (config_valgrind && unlikely(in_valgrind)) old_rzsize = u2rz(old_usize); @@ -1865,7 +1857,7 @@ je_dallocx(void *ptr, int flags) try_tcache = true; UTRACE(ptr, 0, 0); - ifree(tsd_tryget(), ptr, try_tcache); + ifree(tsd_fetch(), ptr, try_tcache); } JEMALLOC_ALWAYS_INLINE_C size_t @@ -1901,7 +1893,7 @@ je_sdallocx(void *ptr, size_t size, int flags) try_tcache = true; UTRACE(ptr, 0, 0); - isfree(tsd_tryget(), ptr, usize, try_tcache); + isfree(tsd_fetch(), ptr, usize, try_tcache); } size_t diff --git a/src/prof.c b/src/prof.c index 5b979989..262f0baa 100644 --- a/src/prof.c +++ b/src/prof.c @@ -850,8 +850,7 @@ prof_bt_count(void) tsd_t *tsd; prof_tdata_t *tdata; - if ((tsd = tsd_tryget()) == NULL) - return (0); + tsd = tsd_fetch(); tdata = prof_tdata_get(tsd, false); if (tdata == NULL) return (0); @@ -1475,8 +1474,7 @@ prof_fdump(void) if (!prof_booted) return; - if ((tsd = tsd_tryget()) == NULL) - return; + tsd = tsd_fetch(); if (opt_prof_final && opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); @@ -1497,8 +1495,7 @@ prof_idump(void) if (!prof_booted) return; - if ((tsd = tsd_tryget()) == NULL) - return; + tsd = tsd_fetch(); tdata = prof_tdata_get(tsd, false); if (tdata == NULL) return; @@ -1526,8 +1523,7 @@ prof_mdump(const char *filename) if (!opt_prof || !prof_booted) return (true); - if ((tsd = tsd_tryget()) == NULL) - return (true); + tsd = tsd_fetch(); if (filename == NULL) { /* No filename specified, so automatically generate one. */ @@ -1553,8 +1549,7 @@ prof_gdump(void) if (!prof_booted) return; - if ((tsd = tsd_tryget()) == NULL) - return; + tsd = tsd_fetch(); tdata = prof_tdata_get(tsd, false); if (tdata == NULL) return; @@ -1677,6 +1672,7 @@ prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata) { assert(prof_tdata_should_destroy(tdata)); + assert(tsd_prof_tdata_get(tsd) != tdata); tdata_tree_remove(&tdatas, tdata); @@ -1704,6 +1700,7 @@ prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) if (tdata->attached) { tdata->attached = false; destroy_tdata = prof_tdata_should_destroy(tdata); + tsd_prof_tdata_set(tsd, NULL); } else destroy_tdata = false; malloc_mutex_unlock(tdata->lock); @@ -1819,8 +1816,7 @@ prof_thread_name_get(void) tsd_t *tsd; prof_tdata_t *tdata; - if ((tsd = tsd_tryget()) == NULL) - return (""); + tsd = tsd_fetch(); tdata = prof_tdata_get(tsd, true); if (tdata == NULL) return (""); @@ -1886,8 +1882,7 @@ prof_thread_active_get(void) tsd_t *tsd; prof_tdata_t *tdata; - if ((tsd = tsd_tryget()) == NULL) - return (false); + tsd = tsd_fetch(); tdata = prof_tdata_get(tsd, true); if (tdata == NULL) return (false); @@ -1900,8 +1895,7 @@ prof_thread_active_set(bool active) tsd_t *tsd; prof_tdata_t *tdata; - if ((tsd = tsd_tryget()) == NULL) - return (true); + tsd = tsd_fetch(); tdata = prof_tdata_get(tsd, true); if (tdata == NULL) return (true); @@ -1988,8 +1982,7 @@ prof_boot2(void) if (malloc_mutex_init(&prof_thread_active_init_mtx)) return (true); - if ((tsd = tsd_tryget()) == NULL) - return (true); + tsd = tsd_fetch(); if (ckh_new(tsd, &bt2gctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) return (true); diff --git a/src/tcache.c b/src/tcache.c index 6f3408cd..07167b6d 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -263,7 +263,8 @@ tcache_get_hard(tsd_t *tsd) { if (!tcache_enabled_get()) { - tcache_enabled_set(false); /* Memoize. */ + if (tsd_nominal(tsd)) + tcache_enabled_set(false); /* Memoize. */ return (NULL); } return (tcache_create(choose_arena(tsd, NULL))); diff --git a/src/tsd.c b/src/tsd.c index 27a70ee8..cbc64e44 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -74,11 +74,6 @@ tsd_cleanup(void *arg) { tsd_t *tsd = (tsd_t *)arg; - if (tsd == NULL) { - /* OOM during re-initialization. */ - return; - } - switch (tsd->state) { case tsd_state_nominal: #define O(n, t) \ diff --git a/test/unit/ckh.c b/test/unit/ckh.c index 03b4f716..c2126487 100644 --- a/test/unit/ckh.c +++ b/test/unit/ckh.c @@ -5,8 +5,7 @@ TEST_BEGIN(test_new_delete) tsd_t *tsd; ckh_t ckh; - tsd = tsd_tryget(); - assert_ptr_not_null(tsd, "Unexpected tsd failure"); + tsd = tsd_fetch(); assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp), "Unexpected ckh_new() error"); @@ -31,8 +30,7 @@ TEST_BEGIN(test_count_insert_search_remove) const char *missing = "A string not in the hash table."; size_t i; - tsd = tsd_tryget(); - assert_ptr_not_null(tsd, "Unexpected tsd failure"); + tsd = tsd_fetch(); assert_false(ckh_new(tsd, &ckh, 2, ckh_string_hash, ckh_string_keycomp), "Unexpected ckh_new() error"); @@ -116,8 +114,7 @@ TEST_BEGIN(test_insert_iter_remove) void *q, *r; size_t i; - tsd = tsd_tryget(); - assert_ptr_not_null(tsd, "Unexpected tsd failure"); + tsd = tsd_fetch(); assert_false(ckh_new(tsd, &ckh, 2, ckh_pointer_hash, ckh_pointer_keycomp), "Unexpected ckh_new() error"); diff --git a/test/unit/tsd.c b/test/unit/tsd.c index 391a7807..eb1c5976 100644 --- a/test/unit/tsd.c +++ b/test/unit/tsd.c @@ -6,17 +6,46 @@ typedef unsigned int data_t; static bool data_cleanup_executed; +malloc_tsd_protos(, data_, data_t) + void data_cleanup(void *arg) { data_t *data = (data_t *)arg; - assert_x_eq(*data, THREAD_DATA, - "Argument passed into cleanup function should match tsd value"); + if (!data_cleanup_executed) { + assert_x_eq(*data, THREAD_DATA, + "Argument passed into cleanup function should match tsd " + "value"); + } data_cleanup_executed = true; + + /* + * Allocate during cleanup for two rounds, in order to assure that + * jemalloc's internal tsd reinitialization happens. + */ + switch (*data) { + case THREAD_DATA: + *data = 1; + data_tsd_set(data); + break; + case 1: + *data = 2; + data_tsd_set(data); + break; + case 2: + return; + default: + not_reached(); + } + + { + void *p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpeced mallocx() failure"); + dallocx(p, 0); + } } -malloc_tsd_protos(, data_, data_t) malloc_tsd_externs(data_, data_t) #define DATA_INIT 0x12345678 malloc_tsd_data(, data_, data_t, DATA_INIT) From 29146e9d15250be0b05cb92550a61a6511e58f79 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 11:23:13 -0700 Subject: [PATCH 130/352] Don't force TLS on behalf of heap profiling. Revert 6716aa83526b3f866d73a033970cc920bc61c13f (Force use of TLS if heap profiling is enabled.). No existing tests indicate that this is necessary, nor does code inspection uncover any potential issues. Most likely the original commit covered up a bug related to tsd-internal allocation that has since been fixed. --- configure.ac | 5 ----- 1 file changed, 5 deletions(-) diff --git a/configure.ac b/configure.ac index 1ee2ed8e..e4afe889 100644 --- a/configure.ac +++ b/configure.ac @@ -793,11 +793,6 @@ fi AC_MSG_CHECKING([configured backtracing method]) AC_MSG_RESULT([$backtrace_method]) if test "x$enable_prof" = "x1" ; then - if test "x${force_tls}" = "x0" ; then - AC_MSG_ERROR([Heap profiling requires TLS]); - fi - force_tls="1" - if test "x$abi" != "xpecoff"; then dnl Heap profiling uses the log(3) function. LIBS="$LIBS -lm" From 34e85b4182d5ae029b558aae3da25fff7c3efe12 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 11:26:05 -0700 Subject: [PATCH 131/352] Make prof-related inline functions always-inline. --- include/jemalloc/internal/prof.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 0ec7c18a..c8014717 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -308,7 +308,7 @@ void prof_free(tsd_t *tsd, const void *ptr, size_t usize); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) -JEMALLOC_INLINE bool +JEMALLOC_ALWAYS_INLINE bool prof_active_get_unlocked(void) { @@ -321,7 +321,7 @@ prof_active_get_unlocked(void) return (prof_active); } -JEMALLOC_INLINE prof_tdata_t * +JEMALLOC_ALWAYS_INLINE prof_tdata_t * prof_tdata_get(tsd_t *tsd, bool create) { prof_tdata_t *tdata; @@ -345,7 +345,7 @@ prof_tdata_get(tsd_t *tsd, bool create) return (tdata); } -JEMALLOC_INLINE prof_tctx_t * +JEMALLOC_ALWAYS_INLINE prof_tctx_t * prof_tctx_get(const void *ptr) { prof_tctx_t *ret; @@ -364,7 +364,7 @@ prof_tctx_get(const void *ptr) return (ret); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { arena_chunk_t *chunk; @@ -380,7 +380,7 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx) huge_prof_tctx_set(ptr, tctx); } -JEMALLOC_INLINE bool +JEMALLOC_ALWAYS_INLINE bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, prof_tdata_t **tdata_out) { @@ -410,7 +410,7 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, } } -JEMALLOC_INLINE prof_tctx_t * +JEMALLOC_ALWAYS_INLINE prof_tctx_t * prof_alloc_prep(tsd_t *tsd, size_t usize, bool update) { prof_tctx_t *ret; @@ -431,7 +431,7 @@ prof_alloc_prep(tsd_t *tsd, size_t usize, bool update) return (ret); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) { @@ -445,7 +445,7 @@ prof_malloc(const void *ptr, size_t usize, prof_tctx_t *tctx) prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, bool updated, size_t old_usize, prof_tctx_t *old_tctx) { @@ -475,7 +475,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, prof_tctx_set(ptr, (prof_tctx_t *)(uintptr_t)1U); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void prof_free(tsd_t *tsd, const void *ptr, size_t usize) { prof_tctx_t *tctx = prof_tctx_get(ptr); From 0800afd03f6f4bc2d722bffedb3398d8ac762c5f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 14:59:17 -0700 Subject: [PATCH 132/352] Silence a compiler warning. --- src/jemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 4a3d968b..3490ecdf 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1262,7 +1262,7 @@ void * je_realloc(void *ptr, size_t size) { void *ret; - tsd_t *tsd; + tsd_t *tsd JEMALLOC_CC_SILENCE_INIT(NULL); size_t usize JEMALLOC_CC_SILENCE_INIT(0); size_t old_usize = 0; UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); From 47395a1b4c0793f676b89a763daaed1cbb510529 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 14:59:38 -0700 Subject: [PATCH 133/352] Avoid purging in microbench when lazy-lock is enabled. --- test/stress/microbench.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/stress/microbench.c b/test/stress/microbench.c index a8267c39..980eca41 100644 --- a/test/stress/microbench.c +++ b/test/stress/microbench.c @@ -19,6 +19,13 @@ compare_funcs(uint64_t nwarmup, uint64_t niter, const char *name_a, { timedelta_t timer_a, timer_b; char ratio_buf[6]; + void *p; + + p = mallocx(1, 0); + if (p == NULL) { + test_fail("Unexpected mallocx() failure"); + return; + } time_func(&timer_a, nwarmup, niter, func_a); time_func(&timer_b, nwarmup, niter, func_b); @@ -28,6 +35,8 @@ compare_funcs(uint64_t nwarmup, uint64_t niter, const char *name_a, "%s=%"PRIu64"us, ratio=1:%s\n", niter, name_a, timer_usec(&timer_a), name_b, timer_usec(&timer_b), ratio_buf); + + dallocx(p, 0); } static void From 16854ebeb77c9403ebd1b85fdd46ee80bb3f3e9d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 15:00:51 -0700 Subject: [PATCH 134/352] Don't disable tcache for lazy-lock. Don't disable tcache when lazy-lock is configured. There already exists a mechanism to disable tcache, but doing so automatically due to lazy-lock causes surprising performance behavior. --- include/jemalloc/internal/tcache.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 1a70972c..1b1d8d98 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -188,8 +188,6 @@ tcache_get(tsd_t *tsd, bool create) if (!config_tcache) return (NULL); - if (config_lazy_lock && !isthreaded) - return (NULL); tcache = tsd_tcache_get(tsd); if (!create) From f04a0bef99e67e11b687a661d6f04e1d7e3bde1f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 4 Oct 2014 15:03:49 -0700 Subject: [PATCH 135/352] Fix prof regressions. Fix prof regressions related to tdata (main per thread profiling data structure) destruction: - Deadlock. The fix for this was intended to be part of 20c31deaae38ed9aa4fe169ed65e0c45cd542955 (Test prof.reset mallctl and fix numerous discovered bugs.) but the fix was left incomplete. - Destruction race. Detaching tdata just prior to destruction without holding the tdatas lock made it possible for another thread to destroy the tdata out from under the thread that was on its way to doing so. --- src/prof.c | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/src/prof.c b/src/prof.c index 262f0baa..a6cea92f 100644 --- a/src/prof.c +++ b/src/prof.c @@ -116,8 +116,10 @@ static bool prof_booted = false; static bool prof_tctx_should_destroy(prof_tctx_t *tctx); static void prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx); -static bool prof_tdata_should_destroy(prof_tdata_t *tdata); -static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata); +static bool prof_tdata_should_destroy(prof_tdata_t *tdata, + bool even_if_attached); +static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, + bool even_if_attached); static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name); /******************************************************************************/ @@ -616,7 +618,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) assert(tctx->cnts.accumbytes == 0); ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); - destroy_tdata = prof_tdata_should_destroy(tdata); + destroy_tdata = prof_tdata_should_destroy(tdata, false); malloc_mutex_unlock(tdata->lock); malloc_mutex_lock(gctx->lock); @@ -644,7 +646,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) prof_gctx_try_destroy(tsd, gctx, tdata); if (destroy_tdata) - prof_tdata_destroy(tsd, tdata); + prof_tdata_destroy(tsd, tdata, false); idalloc(tsd, tctx); } @@ -1656,10 +1658,10 @@ prof_tdata_init(tsd_t *tsd) /* tdata->lock must be held. */ static bool -prof_tdata_should_destroy(prof_tdata_t *tdata) +prof_tdata_should_destroy(prof_tdata_t *tdata, bool even_if_attached) { - if (tdata->attached) + if (tdata->attached && !even_if_attached) return (false); if (ckh_count(&tdata->bt2tctx) != 0) return (false); @@ -1668,10 +1670,11 @@ prof_tdata_should_destroy(prof_tdata_t *tdata) /* tdatas_mtx must be held. */ static void -prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata) +prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, + bool even_if_attached) { - assert(prof_tdata_should_destroy(tdata)); + assert(prof_tdata_should_destroy(tdata, even_if_attached)); assert(tsd_prof_tdata_get(tsd) != tdata); tdata_tree_remove(&tdatas, tdata); @@ -1683,11 +1686,11 @@ prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata) } static void -prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata) +prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) { malloc_mutex_lock(&tdatas_mtx); - prof_tdata_destroy_locked(tsd, tdata); + prof_tdata_destroy_locked(tsd, tdata, even_if_attached); malloc_mutex_unlock(&tdatas_mtx); } @@ -1698,14 +1701,19 @@ prof_tdata_detach(tsd_t *tsd, prof_tdata_t *tdata) malloc_mutex_lock(tdata->lock); if (tdata->attached) { - tdata->attached = false; - destroy_tdata = prof_tdata_should_destroy(tdata); + destroy_tdata = prof_tdata_should_destroy(tdata, true); + /* + * Only detach if !destroy_tdata, because detaching would allow + * another thread to win the race to destroy tdata. + */ + if (!destroy_tdata) + tdata->attached = false; tsd_prof_tdata_set(tsd, NULL); } else destroy_tdata = false; malloc_mutex_unlock(tdata->lock); if (destroy_tdata) - prof_tdata_destroy(tsd, tdata); + prof_tdata_destroy(tsd, tdata, true); } prof_tdata_t * @@ -1731,7 +1739,7 @@ prof_tdata_expire(prof_tdata_t *tdata) if (!tdata->expired) { tdata->expired = true; destroy_tdata = tdata->attached ? false : - prof_tdata_should_destroy(tdata); + prof_tdata_should_destroy(tdata, false); } else destroy_tdata = false; malloc_mutex_unlock(tdata->lock); @@ -1764,8 +1772,7 @@ prof_reset(tsd_t *tsd, size_t lg_sample) prof_tdata_reset_iter, NULL); if (to_destroy != NULL) { next = tdata_tree_next(&tdatas, to_destroy); - tdata_tree_remove(&tdatas, to_destroy); - prof_tdata_destroy(tsd, to_destroy); + prof_tdata_destroy_locked(tsd, to_destroy, false); } else next = NULL; } while (next != NULL); From e9a3fa2e091a48df272e6a7d5d3e92b1a12c489b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 5 Oct 2014 12:05:37 -0700 Subject: [PATCH 136/352] Add missing header includes in jemalloc/jemalloc.h . Add stdlib.h, stdbool.h, and stdint.h to jemalloc/jemalloc.h so that applications only have to #include . This resolves #132. --- doc/jemalloc.xml.in | 3 +-- include/jemalloc/jemalloc_macros.h.in | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 6abb50bc..fcbb4722 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -57,8 +57,7 @@ SYNOPSIS - #include <stdlib.h> -#include <jemalloc/jemalloc.h> + #include <jemalloc/jemalloc.h> Standard API diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in index 1530f9ca..99f12611 100644 --- a/include/jemalloc/jemalloc_macros.h.in +++ b/include/jemalloc/jemalloc_macros.h.in @@ -1,3 +1,6 @@ +#include +#include +#include #include #include From f11a6776c78a09059f8418b718c996a065b33fca Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 5 Oct 2014 13:05:10 -0700 Subject: [PATCH 137/352] Fix OOM-related regression in arena_tcache_fill_small(). Fix an OOM-related regression in arena_tcache_fill_small() that caused cache corruption that would almost certainly expose the application to undefined behavior, usually in the form of an allocation request returning an already-allocated region, or somewhat less likely, a freed region that had already been returned to the arena, thus making it available to the arena for any purpose. This regression was introduced by 9c43c13a35220c10d97a886616899189daceb359 (Reverse tcache fill order.), and was present in all releases from 2.2.0 through 3.6.0. This resolves #98. --- src/arena.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index 79fea728..c223946a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1330,8 +1330,19 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); else ptr = arena_bin_malloc_hard(arena, bin); - if (ptr == NULL) + if (ptr == NULL) { + /* + * OOM. tbin->avail isn't yet filled down to its first + * element, so the successful allocations (if any) must + * be moved to the base of tbin->avail before bailing + * out. + */ + if (i > 0) { + memmove(tbin->avail, &tbin->avail[nfill - i], + i * sizeof(void *)); + } break; + } if (config_fill && unlikely(opt_junk)) { arena_alloc_junk_small(ptr, &arena_bin_info[binind], true); From a95018ee819abf897562d9d1f3bc31d4dd725a8d Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Sat, 4 Oct 2014 01:39:32 -0400 Subject: [PATCH 138/352] Attempt to expand huge allocations in-place. This adds support for expanding huge allocations in-place by requesting memory at a specific address from the chunk allocator. It's currently only implemented for the chunk recycling path, although in theory it could also be done by optimistically allocating new chunks. On Linux, it could attempt an in-place mremap. However, that won't work in practice since the heap is grown downwards and memory is not unmapped (in a normal build, at least). Repeated vector reallocation micro-benchmark: #include #include int main(void) { for (size_t i = 0; i < 100; i++) { void *ptr = NULL; size_t old_size = 0; for (size_t size = 4; size < (1 << 30); size *= 2) { ptr = realloc(ptr, size); if (!ptr) return 1; memset(ptr + old_size, 0xff, size - old_size); old_size = size; } free(ptr); } } The glibc allocator fails to do any in-place reallocations on this benchmark once it passes the M_MMAP_THRESHOLD (default 128k) but it elides the cost of copies via mremap, which is currently not something that jemalloc can use. With this improvement, jemalloc still fails to do any in-place huge reallocations for the first outer loop, but then succeeds 100% of the time for the remaining 99 iterations. The time spent doing allocations and copies drops down to under 5%, with nearly all of it spent doing purging + faulting (when huge pages are disabled) and the array memset. An improved mremap API (MREMAP_RETAIN - #138) would be far more general but this is a portable optimization and would still be useful on Linux for xallocx. Numbers with transparent huge pages enabled: glibc (copies elided via MREMAP_MAYMOVE): 8.471s jemalloc: 17.816s jemalloc + no-op madvise: 13.236s jemalloc + this commit: 6.787s jemalloc + this commit + no-op madvise: 6.144s Numbers with transparent huge pages disabled: glibc (copies elided via MREMAP_MAYMOVE): 15.403s jemalloc: 39.456s jemalloc + no-op madvise: 12.768s jemalloc + this commit: 15.534s jemalloc + this commit + no-op madvise: 6.354s Closes #137 --- doc/jemalloc.xml.in | 7 +- include/jemalloc/internal/arena.h | 4 +- include/jemalloc/internal/chunk.h | 8 +- include/jemalloc/internal/huge.h | 2 +- .../jemalloc/internal/jemalloc_internal.h.in | 2 +- include/jemalloc/jemalloc_typedefs.h.in | 2 +- src/arena.c | 8 +- src/chunk.c | 47 +++++++----- src/huge.c | 74 ++++++++++++++++++- test/integration/chunk.c | 5 +- 10 files changed, 118 insertions(+), 41 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index fcbb4722..f9d464ce 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1351,6 +1351,7 @@ malloc_conf = "xmalloc:true";]]> function that knows how to deallocate the chunks. typedef void *(chunk_alloc_t) + void *chunk size_t size size_t alignment bool *zero @@ -1367,8 +1368,10 @@ malloc_conf = "xmalloc:true";]]> size parameter is always a multiple of the chunk size. The alignment parameter is always a power of two at least as large as the chunk size. Zeroing is mandatory if - *zero is true upon function - entry. + *zero is true upon function entry. If + chunk is not NULL, the + returned pointer must be chunk or + NULL if it could not be allocated. Note that replacing the default chunk allocation function makes the arena's chunk_dalloc; malloc_mutex_unlock(&arena->lock); chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, - arena->ind, size, alignment, zero); + arena->ind, NULL, size, alignment, zero); malloc_mutex_lock(&arena->lock); if (config_stats && chunk != NULL) arena->stats.mapped += chunksize; @@ -459,8 +459,8 @@ arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, } void * -arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, - bool *zero) +arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t size, + size_t alignment, bool *zero) { void *ret; chunk_alloc_t *chunk_alloc; @@ -480,7 +480,7 @@ arena_chunk_alloc_huge(arena_t *arena, size_t size, size_t alignment, malloc_mutex_unlock(&arena->lock); ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, - size, alignment, zero); + new_addr, size, alignment, zero); if (config_stats) { if (ret != NULL) stats_cactive_add(size); diff --git a/src/chunk.c b/src/chunk.c index cde8606e..32b8b3a6 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -42,8 +42,8 @@ static void chunk_dalloc_core(void *chunk, size_t size); /******************************************************************************/ static void * -chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, - size_t alignment, bool base, bool *zero) +chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, + void *new_addr, size_t size, size_t alignment, bool base, bool *zero) { void *ret; extent_node_t *node; @@ -65,11 +65,11 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); - key.addr = NULL; + key.addr = new_addr; key.size = alloc_size; malloc_mutex_lock(&chunks_mtx); node = extent_tree_szad_nsearch(chunks_szad, &key); - if (node == NULL) { + if (node == NULL || (new_addr && node->addr != new_addr)) { malloc_mutex_unlock(&chunks_mtx); return (NULL); } @@ -142,8 +142,8 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, * them if they are returned. */ static void * -chunk_alloc_core(size_t size, size_t alignment, bool base, bool *zero, - dss_prec_t dss_prec) +chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, + bool *zero, dss_prec_t dss_prec) { void *ret; @@ -154,24 +154,30 @@ chunk_alloc_core(size_t size, size_t alignment, bool base, bool *zero, /* "primary" dss. */ if (have_dss && dss_prec == dss_prec_primary) { - if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, - alignment, base, zero)) != NULL) + if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, + new_addr, size, alignment, base, zero)) != NULL) return (ret); - if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) + /* requesting an address only implemented for recycle */ + if (new_addr == NULL + && (ret = chunk_alloc_dss(size, alignment, zero)) != NULL) return (ret); } /* mmap. */ - if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size, - alignment, base, zero)) != NULL) + if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, new_addr, + size, alignment, base, zero)) != NULL) return (ret); - if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) + /* requesting an address only implemented for recycle */ + if (new_addr == NULL && + (ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) return (ret); /* "secondary" dss. */ if (have_dss && dss_prec == dss_prec_secondary) { - if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, - alignment, base, zero)) != NULL) + if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, + new_addr, size, alignment, base, zero)) != NULL) return (ret); - if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) + /* requesting an address only implemented for recycle */ + if (new_addr == NULL && + (ret = chunk_alloc_dss(size, alignment, zero)) != NULL) return (ret); } @@ -219,7 +225,7 @@ chunk_alloc_base(size_t size) bool zero; zero = false; - ret = chunk_alloc_core(size, chunksize, true, &zero, + ret = chunk_alloc_core(NULL, size, chunksize, true, &zero, chunk_dss_prec_get()); if (ret == NULL) return (NULL); @@ -232,11 +238,12 @@ chunk_alloc_base(size_t size) void * chunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc, - unsigned arena_ind, size_t size, size_t alignment, bool *zero) + unsigned arena_ind, void *new_addr, size_t size, size_t alignment, + bool *zero) { void *ret; - ret = chunk_alloc(size, alignment, zero, arena_ind); + ret = chunk_alloc(new_addr, size, alignment, zero, arena_ind); if (ret != NULL && chunk_register(ret, size, false)) { chunk_dalloc(ret, size, arena_ind); ret = NULL; @@ -247,11 +254,11 @@ chunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc, /* Default arena chunk allocation routine in the absence of user override. */ void * -chunk_alloc_default(size_t size, size_t alignment, bool *zero, +chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind) { - return (chunk_alloc_core(size, alignment, false, zero, + return (chunk_alloc_core(new_addr, size, alignment, false, zero, arenas[arena_ind]->dss_prec)); } diff --git a/src/huge.c b/src/huge.c index 2f059b4d..6bdc0767 100644 --- a/src/huge.c +++ b/src/huge.c @@ -47,7 +47,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, */ is_zeroed = zero; arena = choose_arena(tsd, arena); - ret = arena_chunk_alloc_huge(arena, csize, alignment, &is_zeroed); + ret = arena_chunk_alloc_huge(arena, NULL, csize, alignment, &is_zeroed); if (ret == NULL) { base_node_dalloc(node); return (NULL); @@ -95,8 +95,66 @@ huge_dalloc_junk(void *ptr, size_t usize) huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); #endif +static bool +huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { + size_t csize; + void *expand_addr; + size_t expand_size; + extent_node_t *node, key; + arena_t *arena; + bool is_zeroed; + void *ret; + + csize = CHUNK_CEILING(size); + if (csize == 0) { + /* size is large enough to cause size_t wrap-around. */ + return (true); + } + + expand_addr = ptr + oldsize; + expand_size = csize - oldsize; + + malloc_mutex_lock(&huge_mtx); + + key.addr = ptr; + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + + /* Find the current arena. */ + arena = node->arena; + + malloc_mutex_unlock(&huge_mtx); + + /* + * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that + * it is possible to make correct junk/zero fill decisions below. + */ + is_zeroed = zero; + ret = arena_chunk_alloc_huge(arena, expand_addr, expand_size, chunksize, + &is_zeroed); + if (ret == NULL) + return (true); + + assert(ret == expand_addr); + + malloc_mutex_lock(&huge_mtx); + /* Update the size of the huge allocation. */ + node->size = csize; + malloc_mutex_unlock(&huge_mtx); + + if (config_fill && !zero) { + if (unlikely(opt_junk)) + memset(expand_addr, 0xa5, expand_size); + else if (unlikely(opt_zero) && !is_zeroed) + memset(expand_addr, 0, expand_size); + } + return (false); +} + bool -huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) +huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, + bool zero) { /* Both allocations must be huge to avoid a move. */ @@ -145,7 +203,15 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) return (false); } - return (true); + /* Attempt to expand the allocation in-place. */ + if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, zero)) { + if (extra == 0) + return (true); + + /* Try again, this time without extra. */ + return (huge_ralloc_no_move_expand(ptr, oldsize, size, zero)); + } + return (false); } void * @@ -156,7 +222,7 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t copysize; /* Try to avoid moving the allocation. */ - if (!huge_ralloc_no_move(ptr, oldsize, size, extra)) + if (!huge_ralloc_no_move(ptr, oldsize, size, extra, zero)) return (ptr); /* diff --git a/test/integration/chunk.c b/test/integration/chunk.c index 28537098..89938504 100644 --- a/test/integration/chunk.c +++ b/test/integration/chunk.c @@ -11,10 +11,11 @@ chunk_dalloc(void *chunk, size_t size, unsigned arena_ind) } void * -chunk_alloc(size_t size, size_t alignment, bool *zero, unsigned arena_ind) +chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, + unsigned arena_ind) { - return (old_alloc(size, alignment, zero, arena_ind)); + return (old_alloc(new_addr, size, alignment, zero, arena_ind)); } TEST_BEGIN(test_chunk) From 3c3b3b1a94705c8019b973fb679dd99bd19305af Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 5 Oct 2014 14:48:44 -0700 Subject: [PATCH 139/352] Fix a docbook element nesting nit. According to the docbook documentation for , its parent must be ; fix accordingly. Nonetheless, the man page processor fails badly when this construct is embedded in a (which is documented to be legal), although the html processor does fine. --- doc/jemalloc.xml.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index f9d464ce..1f692f78 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1349,14 +1349,14 @@ malloc_conf = "xmalloc:true";]]> also be set via arena.<i>.chunk.dalloc to a companion function that knows how to deallocate the chunks. - + typedef void *(chunk_alloc_t) void *chunk size_t size size_t alignment bool *zero unsigned arena_ind - + A chunk allocation function conforms to the chunk_alloc_t type and upon success returns a pointer to size bytes of memory on behalf of arena arena_ind such @@ -1397,12 +1397,12 @@ malloc_conf = "xmalloc:true";]]> arena creation), but the automatically created arenas may have already created chunks prior to the application having an opportunity to take over chunk allocation. - + typedef void (chunk_dalloc_t) void *chunk size_t size unsigned arena_ind - + A chunk deallocation function conforms to the chunk_dalloc_t type and deallocates a chunk of given size on From 155bfa7da18cab0d21d87aa2dce4554166836f5d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 5 Oct 2014 17:54:10 -0700 Subject: [PATCH 140/352] Normalize size classes. Normalize size classes to use the same number of size classes per size doubling (currently hard coded to 4), across the intire range of size classes. Small size classes already used this spacing, but in order to support this change, additional small size classes now fill [4 KiB .. 16 KiB). Large size classes range from [16 KiB .. 4 MiB). Huge size classes now support non-multiples of the chunk size in order to fill (4 MiB .. 16 MiB). --- include/jemalloc/internal/arena.h | 231 +++----------- include/jemalloc/internal/chunk.h | 3 - include/jemalloc/internal/huge.h | 2 +- .../jemalloc/internal/jemalloc_internal.h.in | 299 +++++++++++++----- include/jemalloc/internal/private_symbols.txt | 22 +- include/jemalloc/internal/size_classes.sh | 15 +- include/jemalloc/internal/stats.h | 7 +- include/jemalloc/internal/tcache.h | 52 +-- src/arena.c | 223 ++++++------- src/chunk.c | 3 - src/ctl.c | 2 +- src/huge.c | 113 +++++-- src/jemalloc.c | 34 +- src/tcache.c | 8 +- test/unit/junk.c | 17 +- test/unit/mallctl.c | 2 +- 16 files changed, 558 insertions(+), 475 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 1f985723..681b5802 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -1,6 +1,8 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES +#define LARGE_MINCLASS (ZU(1) << LG_LARGE_MINCLASS) + /* Maximum number of regions in one run. */ #define LG_RUN_MAXREGS (LG_PAGE - LG_TINY_MIN) #define RUN_MAXREGS (1U << LG_RUN_MAXREGS) @@ -96,11 +98,15 @@ struct arena_chunk_map_bits_s { * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx * -------- -------- ----++++ ++++D-LA * - * Large (sampled, size <= PAGE): + * Large (sampled, size <= LARGE_MINCLASS): * ssssssss ssssssss ssssnnnn nnnnD-LA + * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + * -------- -------- ----++++ ++++D-LA * - * Large (not sampled, size == PAGE): + * Large (not sampled, size == LARGE_MINCLASS): * ssssssss ssssssss ssss++++ ++++D-LA + * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + * -------- -------- ----++++ ++++D-LA */ size_t bits; #define CHUNK_MAP_BININD_SHIFT 4 @@ -325,30 +331,21 @@ struct arena_s { #ifdef JEMALLOC_H_EXTERNS extern ssize_t opt_lg_dirty_mult; -/* - * small_size2bin_tab is a compact lookup table that rounds request sizes up to - * size classes. In order to reduce cache footprint, the table is compressed, - * and all accesses are via small_size2bin(). - */ -extern uint8_t const small_size2bin_tab[]; -/* - * small_bin2size_tab duplicates information in arena_bin_info, but in a const - * array, for which it is easier for the compiler to optimize repeated - * dereferences. - */ -extern uint32_t const small_bin2size_tab[NBINS]; extern arena_bin_info_t arena_bin_info[NBINS]; -/* Number of large size classes. */ -#define nlclasses (chunk_npages - map_bias) +extern size_t map_bias; /* Number of arena chunk header pages. */ +extern size_t map_misc_offset; +extern size_t arena_maxrun; /* Max run size for arenas. */ +extern size_t arena_maxclass; /* Max size class for arenas. */ +extern size_t nlclasses; /* Number of large size classes. */ void *arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero); void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size); void arena_purge_all(arena_t *arena); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, - size_t binind, uint64_t prof_accumbytes); + index_t binind, uint64_t prof_accumbytes); void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero); #ifdef JEMALLOC_JET @@ -403,15 +400,6 @@ void arena_postfork_child(arena_t *arena); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -size_t small_size2bin_compute(size_t size); -size_t small_size2bin_lookup(size_t size); -size_t small_size2bin(size_t size); -size_t small_bin2size_compute(size_t binind); -size_t small_bin2size_lookup(size_t binind); -size_t small_bin2size(size_t binind); -size_t small_s2u_compute(size_t size); -size_t small_s2u_lookup(size_t size); -size_t small_s2u(size_t size); arena_chunk_map_bits_t *arena_bitselm_get(arena_chunk_t *chunk, size_t pageind); arena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, @@ -426,7 +414,7 @@ size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); -size_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); +index_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); @@ -439,16 +427,16 @@ void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t flags); void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, - size_t binind); + index_t binind); void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, - size_t runind, size_t binind, size_t flags); + size_t runind, index_t binind, size_t flags); void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, size_t unzeroed); bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum(arena_t *arena, uint64_t accumbytes); -size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); -size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); +index_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); +index_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); prof_tctx_t *arena_prof_tctx_get(const void *ptr); @@ -464,148 +452,6 @@ void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) # ifdef JEMALLOC_ARENA_INLINE_A -JEMALLOC_INLINE size_t -small_size2bin_compute(size_t size) -{ -#if (NTBINS != 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; - size_t lg_ceil = lg_floor(pow2_ceil(size)); - return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); - } else -#endif - { - size_t x = lg_floor((size<<1)-1); - size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : - x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); - size_t grp = shift << LG_SIZE_CLASS_GROUP; - - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; - - size_t delta_inverse_mask = ZI(-1) << lg_delta; - size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); - - size_t bin = NTBINS + grp + mod; - return (bin); - } -} - -JEMALLOC_ALWAYS_INLINE size_t -small_size2bin_lookup(size_t size) -{ - - assert(size <= LOOKUP_MAXCLASS); - { - size_t ret = ((size_t)(small_size2bin_tab[(size-1) >> - LG_TINY_MIN])); - assert(ret == small_size2bin_compute(size)); - return (ret); - } -} - -JEMALLOC_ALWAYS_INLINE size_t -small_size2bin(size_t size) -{ - - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) - return (small_size2bin_lookup(size)); - else - return (small_size2bin_compute(size)); -} - -JEMALLOC_INLINE size_t -small_bin2size_compute(size_t binind) -{ -#if (NTBINS > 0) - if (binind < NTBINS) - return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + binind)); - else -#endif - { - size_t reduced_binind = binind - NTBINS; - size_t grp = reduced_binind >> LG_SIZE_CLASS_GROUP; - size_t mod = reduced_binind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - - 1); - - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_QUANTUM + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; - - size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_QUANTUM-1); - size_t mod_size = (mod+1) << lg_delta; - - size_t usize = grp_size + mod_size; - return (usize); - } -} - -JEMALLOC_ALWAYS_INLINE size_t -small_bin2size_lookup(size_t binind) -{ - - assert(binind < NBINS); - { - size_t ret = (size_t)small_bin2size_tab[binind]; - assert(ret == small_bin2size_compute(binind)); - return (ret); - } -} - -JEMALLOC_ALWAYS_INLINE size_t -small_bin2size(size_t binind) -{ - - return (small_bin2size_lookup(binind)); -} - -JEMALLOC_ALWAYS_INLINE size_t -small_s2u_compute(size_t size) -{ -#if (NTBINS > 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; - size_t lg_ceil = lg_floor(pow2_ceil(size)); - return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : - (ZU(1) << lg_ceil)); - } else -#endif - { - size_t x = lg_floor((size<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (size + delta_mask) & ~delta_mask; - return (usize); - } -} - -JEMALLOC_ALWAYS_INLINE size_t -small_s2u_lookup(size_t size) -{ - size_t ret = small_bin2size(small_size2bin(size)); - - assert(ret == small_s2u_compute(size)); - return (ret); -} - -JEMALLOC_ALWAYS_INLINE size_t -small_s2u(size_t size) -{ - - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) - return (small_s2u_lookup(size)); - else - return (small_s2u_compute(size)); -} -# endif /* JEMALLOC_ARENA_INLINE_A */ - -# ifdef JEMALLOC_ARENA_INLINE_B JEMALLOC_ALWAYS_INLINE arena_chunk_map_bits_t * arena_bitselm_get(arena_chunk_t *chunk, size_t pageind) { @@ -714,11 +560,11 @@ arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) return (mapbits >> LG_PAGE); } -JEMALLOC_ALWAYS_INLINE size_t +JEMALLOC_ALWAYS_INLINE index_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; - size_t binind; + index_t binind; mapbits = arena_mapbits_get(chunk, pageind); binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; @@ -810,20 +656,20 @@ arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, JEMALLOC_ALWAYS_INLINE void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, - size_t binind) + index_t binind) { size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); size_t mapbits = arena_mapbitsp_read(mapbitsp); assert(binind <= BININD_INVALID); - assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); + assert(arena_mapbits_large_size_get(chunk, pageind) == LARGE_MINCLASS); arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | (binind << CHUNK_MAP_BININD_SHIFT)); } JEMALLOC_ALWAYS_INLINE void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, - size_t binind, size_t flags) + index_t binind, size_t flags) { size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); size_t mapbits = arena_mapbitsp_read(mapbitsp); @@ -893,10 +739,10 @@ arena_prof_accum(arena_t *arena, uint64_t accumbytes) } } -JEMALLOC_ALWAYS_INLINE size_t +JEMALLOC_ALWAYS_INLINE index_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits) { - size_t binind; + index_t binind; binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; @@ -908,7 +754,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) size_t rpages_ind; arena_run_t *run; arena_bin_t *bin; - size_t actual_binind; + index_t actual_binind; arena_bin_info_t *bin_info; arena_chunk_map_misc_t *miscelm; void *rpages; @@ -938,13 +784,13 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) return (binind); } -# endif /* JEMALLOC_ARENA_INLINE_B */ +# endif /* JEMALLOC_ARENA_INLINE_A */ -# ifdef JEMALLOC_ARENA_INLINE_C -JEMALLOC_INLINE size_t +# ifdef JEMALLOC_ARENA_INLINE_B +JEMALLOC_INLINE index_t arena_bin_index(arena_t *arena, arena_bin_t *bin) { - size_t binind = bin - arena->bins; + index_t binind = bin - arena->bins; assert(binind < NBINS); return (binind); } @@ -1102,7 +948,8 @@ arena_salloc(const void *ptr, bool demote) { size_t ret; arena_chunk_t *chunk; - size_t pageind, binind; + size_t pageind; + index_t binind; assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); @@ -1122,10 +969,6 @@ arena_salloc(const void *ptr, bool demote) ret = arena_mapbits_large_size_get(chunk, pageind); assert(ret != 0); assert(pageind + (ret>>LG_PAGE) <= chunk_npages); - assert(ret == PAGE || arena_mapbits_large_size_get(chunk, - pageind+(ret>>LG_PAGE)-1) == 0); - assert(binind == arena_mapbits_binind_get(chunk, - pageind+(ret>>LG_PAGE)-1)); assert(arena_mapbits_dirty_get(chunk, pageind) == arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); } else { @@ -1133,7 +976,7 @@ arena_salloc(const void *ptr, bool demote) assert(arena_mapbits_large_get(chunk, pageind) != 0 || arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, pageind)) == binind); - ret = small_bin2size(binind); + ret = index2size(binind); } return (ret); @@ -1155,7 +998,7 @@ arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) /* Small allocation. */ if (likely(try_tcache) && likely((tcache = tcache_get(tsd, false)) != NULL)) { - size_t binind = arena_ptr_small_binind_get(ptr, + index_t binind = arena_ptr_small_binind_get(ptr, mapbits); tcache_dalloc_small(tcache, ptr, binind); } else @@ -1186,7 +1029,7 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, /* Small allocation. */ if (likely(try_tcache) && likely((tcache = tcache_get(tsd, false)) != NULL)) { - size_t binind = small_size2bin(size); + index_t binind = size2index(size); tcache_dalloc_small(tcache, ptr, binind); } else { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> @@ -1203,7 +1046,7 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, arena_dalloc_large(chunk->arena, chunk, ptr); } } -# endif /* JEMALLOC_ARENA_INLINE_C */ +# endif /* JEMALLOC_ARENA_INLINE_B */ #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 2e68a020..764b7aca 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -40,9 +40,6 @@ extern rtree_t *chunks_rtree; extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ extern size_t chunk_npages; -extern size_t map_bias; /* Number of arena chunk header pages. */ -extern size_t map_misc_offset; -extern size_t arena_maxclass; /* Max size class for arenas. */ void *chunk_alloc_base(size_t size); void *chunk_alloc_arena(chunk_alloc_t *chunk_alloc, diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 00d8c09d..939993f2 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -10,7 +10,7 @@ #ifdef JEMALLOC_H_EXTERNS void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero); -void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, +void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, bool zero); bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index a169221b..8f0beb9e 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -165,6 +165,9 @@ static const bool config_ivsalloc = #include "jemalloc/internal/jemalloc_internal_macros.h" +/* Size class index type. */ +typedef unsigned index_t; + #define MALLOCX_ARENA_MASK ((int)~0xff) #define MALLOCX_LG_ALIGN_MASK ((int)0x3f) /* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ @@ -397,6 +400,18 @@ extern arena_t **arenas; extern unsigned narenas_total; extern unsigned narenas_auto; /* Read-only after initialization. */ +/* + * index2size_tab encodes the same information as could be computed (at + * unacceptable cost in some code paths) by index2size_compute(). + */ +extern size_t const index2size_tab[NSIZES]; +/* + * size2index_tab is a compact lookup table that rounds request sizes up to + * size classes. In order to reduce cache footprint, the table is compressed, + * and all accesses are via size2index(). + */ +extern uint8_t const size2index_tab[]; + arena_t *arenas_extend(unsigned ind); arena_t *choose_arena_hard(tsd_t *tsd); void thread_allocated_cleanup(tsd_t *tsd); @@ -449,15 +464,15 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" -/* - * Include arena.h the first time in order to provide inline functions for this - * header's inlines. - */ -#define JEMALLOC_ARENA_INLINE_A -#include "jemalloc/internal/arena.h" -#undef JEMALLOC_ARENA_INLINE_A - #ifndef JEMALLOC_ENABLE_INLINE +index_t size2index_compute(size_t size); +index_t size2index_lookup(size_t size); +index_t size2index(size_t size); +size_t index2size_compute(index_t index); +size_t index2size_lookup(index_t index); +size_t index2size(index_t index); +size_t s2u_compute(size_t size); +size_t s2u_lookup(size_t size); size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment); unsigned narenas_total_get(void); @@ -465,6 +480,135 @@ arena_t *choose_arena(tsd_t *tsd, arena_t *arena); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +JEMALLOC_INLINE index_t +size2index_compute(size_t size) +{ + +#if (NTBINS != 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + size_t lg_ceil = lg_floor(pow2_ceil(size)); + return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); + } else +#endif + { + size_t x = lg_floor((size<<1)-1); + size_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : + x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); + size_t grp = shift << LG_SIZE_CLASS_GROUP; + + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + + size_t delta_inverse_mask = ZI(-1) << lg_delta; + size_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + + size_t index = NTBINS + grp + mod; + return (index); + } +} + +JEMALLOC_ALWAYS_INLINE index_t +size2index_lookup(size_t size) +{ + + assert(size <= LOOKUP_MAXCLASS); + { + size_t ret = ((size_t)(size2index_tab[(size-1) >> + LG_TINY_MIN])); + assert(ret == size2index_compute(size)); + return (ret); + } +} + +JEMALLOC_ALWAYS_INLINE index_t +size2index(size_t size) +{ + + assert(size > 0); + if (likely(size <= LOOKUP_MAXCLASS)) + return (size2index_lookup(size)); + else + return (size2index_compute(size)); +} + +JEMALLOC_INLINE size_t +index2size_compute(index_t index) +{ + +#if (NTBINS > 0) + if (index < NTBINS) + return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); + else +#endif + { + size_t reduced_index = index - NTBINS; + size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; + size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - + 1); + + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_QUANTUM + + (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_QUANTUM-1); + size_t mod_size = (mod+1) << lg_delta; + + size_t usize = grp_size + mod_size; + return (usize); + } +} + +JEMALLOC_ALWAYS_INLINE size_t +index2size_lookup(index_t index) +{ + size_t ret = (size_t)index2size_tab[index]; + assert(ret == index2size_compute(index)); + return (ret); +} + +JEMALLOC_ALWAYS_INLINE size_t +index2size(index_t index) +{ + + assert(index < NSIZES); + return (index2size_lookup(index)); +} + +JEMALLOC_ALWAYS_INLINE size_t +s2u_compute(size_t size) +{ + +#if (NTBINS > 0) + if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { + size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + size_t lg_ceil = lg_floor(pow2_ceil(size)); + return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : + (ZU(1) << lg_ceil)); + } else +#endif + { + size_t x = lg_floor((size<<1)-1); + size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (size + delta_mask) & ~delta_mask; + return (usize); + } +} + +JEMALLOC_ALWAYS_INLINE size_t +s2u_lookup(size_t size) +{ + size_t ret = index2size_lookup(size2index_lookup(size)); + + assert(ret == s2u_compute(size)); + return (ret); +} + /* * Compute usable size that would result from allocating an object with the * specified size. @@ -473,11 +617,11 @@ JEMALLOC_ALWAYS_INLINE size_t s2u(size_t size) { - if (size <= SMALL_MAXCLASS) - return (small_s2u(size)); - if (size <= arena_maxclass) - return (PAGE_CEILING(size)); - return (CHUNK_CEILING(size)); + assert(size > 0); + if (likely(size <= LOOKUP_MAXCLASS)) + return (s2u_lookup(size)); + else + return (s2u_compute(size)); } /* @@ -491,71 +635,78 @@ sa2u(size_t size, size_t alignment) assert(alignment != 0 && ((alignment - 1) & alignment) == 0); - /* - * Round size up to the nearest multiple of alignment. - * - * This done, we can take advantage of the fact that for each small - * size class, every object is aligned at the smallest power of two - * that is non-zero in the base two representation of the size. For - * example: - * - * Size | Base 2 | Minimum alignment - * -----+----------+------------------ - * 96 | 1100000 | 32 - * 144 | 10100000 | 32 - * 192 | 11000000 | 64 - */ - usize = ALIGNMENT_CEILING(size, alignment); - /* - * (usize < size) protects against the combination of maximal - * alignment and size greater than maximal alignment. - */ - if (usize < size) { - /* size_t overflow. */ - return (0); + /* Try for a small size class. */ + if (size <= SMALL_MAXCLASS && alignment < PAGE) { + /* + * Round size up to the nearest multiple of alignment. + * + * This done, we can take advantage of the fact that for each + * small size class, every object is aligned at the smallest + * power of two that is non-zero in the base two representation + * of the size. For example: + * + * Size | Base 2 | Minimum alignment + * -----+----------+------------------ + * 96 | 1100000 | 32 + * 144 | 10100000 | 32 + * 192 | 11000000 | 64 + */ + usize = s2u(ALIGNMENT_CEILING(size, alignment)); + if (usize < LARGE_MINCLASS) + return (usize); } - if (usize <= arena_maxclass && alignment <= PAGE) { - if (usize <= SMALL_MAXCLASS) - return (small_s2u(usize)); - return (PAGE_CEILING(usize)); - } else { - size_t run_size; - + /* Try for a large size class. */ + if (size <= arena_maxclass && alignment < chunksize) { /* * We can't achieve subpage alignment, so round up alignment - * permanently; it makes later calculations simpler. + * to the minimum that can actually be supported. */ alignment = PAGE_CEILING(alignment); - usize = PAGE_CEILING(size); - /* - * (usize < size) protects against very large sizes within - * PAGE of SIZE_T_MAX. - * - * (usize + alignment < usize) protects against the - * combination of maximal alignment and usize large enough - * to cause overflow. This is similar to the first overflow - * check above, but it needs to be repeated due to the new - * usize value, which may now be *equal* to maximal - * alignment, whereas before we only detected overflow if the - * original size was *greater* than maximal alignment. - */ - if (usize < size || usize + alignment < usize) { - /* size_t overflow. */ - return (0); - } + + /* Make sure result is a large size class. */ + usize = (size <= LARGE_MINCLASS) ? LARGE_MINCLASS : s2u(size); /* * Calculate the size of the over-size run that arena_palloc() * would need to allocate in order to guarantee the alignment. - * If the run wouldn't fit within a chunk, round up to a huge - * allocation size. */ - run_size = usize + alignment - PAGE; - if (run_size <= arena_maxclass) - return (PAGE_CEILING(usize)); - return (CHUNK_CEILING(usize)); + if (usize + alignment - PAGE <= arena_maxrun) + return (usize); } + + /* Huge size class. Beware of size_t overflow. */ + + /* + * We can't achieve subchunk alignment, so round up alignment to the + * minimum that can actually be supported. + */ + alignment = CHUNK_CEILING(alignment); + if (alignment == 0) { + /* size_t overflow. */ + return (0); + } + + /* Make sure result is a huge size class. */ + if (size <= chunksize) + usize = chunksize; + else { + usize = s2u(size); + if (usize < size) { + /* size_t overflow. */ + return (0); + } + } + + /* + * Calculate the multi-chunk mapping that huge_palloc() would need in + * order to guarantee the alignment. + */ + if (usize + alignment - PAGE < usize) { + /* size_t overflow. */ + return (0); + } + return (usize); } JEMALLOC_INLINE unsigned @@ -591,16 +742,16 @@ choose_arena(tsd_t *tsd, arena_t *arena) #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/rtree.h" /* - * Include arena.h the second and third times in order to resolve circular - * dependencies with tcache.h. + * Include portions of arena.h interleaved with tcache.h in order to resolve + * circular dependencies. */ +#define JEMALLOC_ARENA_INLINE_A +#include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_INLINE_A +#include "jemalloc/internal/tcache.h" #define JEMALLOC_ARENA_INLINE_B #include "jemalloc/internal/arena.h" #undef JEMALLOC_ARENA_INLINE_B -#include "jemalloc/internal/tcache.h" -#define JEMALLOC_ARENA_INLINE_C -#include "jemalloc/internal/arena.h" -#undef JEMALLOC_ARENA_INLINE_C #include "jemalloc/internal/hash.h" #include "jemalloc/internal/quarantine.h" @@ -678,7 +829,7 @@ ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, assert(usize != 0); assert(usize == sa2u(usize, alignment)); - if (usize <= arena_maxclass && alignment <= PAGE) + if (usize <= SMALL_MAXCLASS && alignment < PAGE) ret = arena_malloc(tsd, arena, usize, zero, try_tcache); else { if (usize <= arena_maxclass) { @@ -742,7 +893,7 @@ u2rz(size_t usize) size_t ret; if (usize <= SMALL_MAXCLASS) { - size_t binind = small_size2bin(usize); + index_t binind = size2index(usize); ret = arena_bin_info[binind].redzone_size; } else ret = 0; diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 4ea9a953..1a7fde4b 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -41,6 +41,7 @@ arena_mapbitsp_get arena_mapbitsp_read arena_mapbitsp_write arena_maxclass +arena_maxrun arena_miscelm_get arena_miscelm_to_pageind arena_miscelm_to_rpages @@ -216,6 +217,10 @@ idalloct imalloc imalloct in_valgrind +index2size +index2size_compute +index2size_lookup +index2size_tab ipalloc ipalloct iqalloc @@ -338,19 +343,14 @@ rtree_postfork_parent rtree_prefork rtree_set s2u +s2u_compute +s2u_lookup sa2u set_errno -small_bin2size -small_bin2size_compute -small_bin2size_lookup -small_bin2size_tab -small_s2u -small_s2u_compute -small_s2u_lookup -small_size2bin -small_size2bin_compute -small_size2bin_lookup -small_size2bin_tab +size2index +size2index_compute +size2index_lookup +size2index_tab stats_cactive stats_cactive_add stats_cactive_get diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 0cfac72d..897570cc 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -61,7 +61,7 @@ size_class() { rem="yes" fi - if [ ${lg_size} -lt ${lg_p} ] ; then + if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then bin="yes" else bin="no" @@ -159,6 +159,7 @@ size_classes() { nbins=$((${index} + 1)) # Final written value is correct: small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" + lg_large_minclass=$((${lg_grp} + 1)) fi index=$((${index} + 1)) ndelta=$((${ndelta} + 1)) @@ -167,14 +168,17 @@ size_classes() { lg_delta=$((${lg_delta} + 1)) done echo + nsizes=${index} # Defined upon completion: # - ntbins # - nlbins # - nbins + # - nsizes # - lg_tiny_maxclass # - lookup_maxclass # - small_maxclass + # - lg_large_minclass } cat <tbins[binind]; - size = small_bin2size(binind); + usize = index2size(binind); ret = tcache_alloc_easy(tbin); if (unlikely(ret == NULL)) { ret = tcache_alloc_small_hard(tcache, tbin, binind); if (ret == NULL) return (NULL); } - assert(tcache_salloc(ret) == size); + assert(tcache_salloc(ret) == usize); if (likely(!zero)) { if (config_fill) { @@ -254,20 +255,20 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) arena_alloc_junk_small(ret, &arena_bin_info[binind], false); } else if (unlikely(opt_zero)) - memset(ret, 0, size); + memset(ret, 0, usize); } } else { if (config_fill && unlikely(opt_junk)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } - memset(ret, 0, size); + memset(ret, 0, usize); } if (config_stats) tbin->tstats.nrequests++; if (config_prof) - tcache->prof_accumbytes += size; + tcache->prof_accumbytes += usize; tcache_event(tcache); return (ret); } @@ -276,12 +277,13 @@ JEMALLOC_ALWAYS_INLINE void * tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) { void *ret; - size_t binind; + index_t binind; + size_t usize; tcache_bin_t *tbin; - size = PAGE_CEILING(size); - assert(size <= tcache_maxclass); - binind = NBINS + (size >> LG_PAGE) - 1; + binind = size2index(size); + usize = index2size(binind); + assert(usize <= tcache_maxclass); assert(binind < nhbins); tbin = &tcache->tbins[binind]; ret = tcache_alloc_easy(tbin); @@ -290,11 +292,11 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) * Only allocate one large object at a time, because it's quite * expensive to create one and not use it. */ - ret = arena_malloc_large(tcache->arena, size, zero); + ret = arena_malloc_large(tcache->arena, usize, zero); if (ret == NULL) return (NULL); } else { - if (config_prof && size == PAGE) { + if (config_prof && usize == LARGE_MINCLASS) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> @@ -305,17 +307,17 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) if (likely(!zero)) { if (config_fill) { if (unlikely(opt_junk)) - memset(ret, 0xa5, size); + memset(ret, 0xa5, usize); else if (unlikely(opt_zero)) - memset(ret, 0, size); + memset(ret, 0, usize); } } else - memset(ret, 0, size); + memset(ret, 0, usize); if (config_stats) tbin->tstats.nrequests++; if (config_prof) - tcache->prof_accumbytes += size; + tcache->prof_accumbytes += usize; } tcache_event(tcache); @@ -323,7 +325,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) } JEMALLOC_ALWAYS_INLINE void -tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind) +tcache_dalloc_small(tcache_t *tcache, void *ptr, index_t binind) { tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; @@ -349,7 +351,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind) JEMALLOC_ALWAYS_INLINE void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) { - size_t binind; + index_t binind; tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; @@ -357,7 +359,7 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) assert(tcache_salloc(ptr) > SMALL_MAXCLASS); assert(tcache_salloc(ptr) <= tcache_maxclass); - binind = NBINS + (size >> LG_PAGE) - 1; + binind = size2index(size); if (config_fill && unlikely(opt_junk)) memset(ptr, 0x5a, size); diff --git a/src/arena.c b/src/arena.c index b7300a92..49a30572 100644 --- a/src/arena.c +++ b/src/arena.c @@ -7,42 +7,11 @@ ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; arena_bin_info_t arena_bin_info[NBINS]; -JEMALLOC_ALIGNED(CACHELINE) -const uint32_t small_bin2size_tab[NBINS] = { -#define B2S_bin_yes(size) \ - size, -#define B2S_bin_no(size) -#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ - B2S_bin_##bin((ZU(1)<> LG_PAGE; size_t mapbits = arena_mapbits_get(chunk, pageind); - size_t binind = arena_ptr_small_binind_get(ptr, mapbits); + index_t binind = arena_ptr_small_binind_get(ptr, mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind = arena_run_regind(run, bin_info, ptr); @@ -375,7 +344,7 @@ arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) static void arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, - size_t binind) + index_t binind) { arena_chunk_t *chunk; arena_chunk_map_misc_t *miscelm; @@ -429,9 +398,9 @@ arena_chunk_init_spare(arena_t *arena) assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == - arena_maxclass); + arena_maxrun); assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == - arena_maxclass); + arena_maxrun); assert(arena_mapbits_dirty_get(chunk, map_bias) == arena_mapbits_dirty_get(chunk, chunk_npages-1)); @@ -518,8 +487,7 @@ arena_chunk_init_hard(arena_t *arena) * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. */ unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; - arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, - unzeroed); + arena_mapbits_unallocated_set(chunk, map_bias, arena_maxrun, unzeroed); /* * There is no need to initialize the internal page map entries unless * the chunk is not zeroed. @@ -544,7 +512,7 @@ arena_chunk_init_hard(arena_t *arena) } } } - arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, + arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxrun, unzeroed); return (chunk); @@ -607,9 +575,9 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == - arena_maxclass); + arena_maxrun); assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == - arena_maxclass); + arena_maxrun); assert(arena_mapbits_dirty_get(chunk, map_bias) == arena_mapbits_dirty_get(chunk, chunk_npages-1)); @@ -682,7 +650,7 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero) } static arena_run_t * -arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) +arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind) { arena_run_t *run; arena_chunk_map_misc_t *miscelm; @@ -700,7 +668,7 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) } static arena_run_t * -arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) +arena_run_alloc_small(arena_t *arena, size_t size, index_t binind) { arena_chunk_t *chunk; arena_run_t *run; @@ -1034,7 +1002,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_mapbits_large_size_get(chunk, run_ind+(size>>LG_PAGE)-1) == 0); } else { - size_t binind = arena_bin_index(arena, run->bin); + index_t binind = arena_bin_index(arena, run->bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; size = bin_info->run_size; } @@ -1079,9 +1047,9 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_dirty_insert(arena, chunk, run_ind, run_pages); /* Deallocate chunk if it is now completely unused. */ - if (size == arena_maxclass) { + if (size == arena_maxrun) { assert(run_ind == map_bias); - assert(run_pages == (arena_maxclass >> LG_PAGE)); + assert(run_pages == (arena_maxrun >> LG_PAGE)); arena_chunk_dalloc(arena, chunk); } @@ -1212,7 +1180,7 @@ static arena_run_t * arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) { arena_run_t *run; - size_t binind; + index_t binind; arena_bin_info_t *bin_info; /* Look for a usable run. */ @@ -1264,7 +1232,7 @@ static void * arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) { void *ret; - size_t binind; + index_t binind; arena_bin_info_t *bin_info; arena_run_t *run; @@ -1310,7 +1278,7 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) } void -arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, +arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, uint64_t prof_accumbytes) { unsigned i, nfill; @@ -1450,14 +1418,14 @@ arena_dalloc_junk_small_t *arena_dalloc_junk_small = void arena_quarantine_junk_small(void *ptr, size_t usize) { - size_t binind; + index_t binind; arena_bin_info_t *bin_info; cassert(config_fill); assert(opt_junk); assert(opt_quarantine); assert(usize <= SMALL_MAXCLASS); - binind = small_size2bin(usize); + binind = size2index(usize); bin_info = &arena_bin_info[binind]; arena_redzones_validate(ptr, bin_info, true); } @@ -1468,12 +1436,12 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) void *ret; arena_bin_t *bin; arena_run_t *run; - size_t binind; + index_t binind; - binind = small_size2bin(size); + binind = size2index(size); assert(binind < NBINS); bin = &arena->bins[binind]; - size = small_bin2size(binind); + size = index2size(binind); malloc_mutex_lock(&bin->lock); if ((run = bin->runcur) != NULL && run->nfree > 0) @@ -1520,14 +1488,15 @@ void * arena_malloc_large(arena_t *arena, size_t size, bool zero) { void *ret; + size_t usize; arena_run_t *run; arena_chunk_map_misc_t *miscelm; UNUSED bool idump; /* Large allocation. */ - size = PAGE_CEILING(size); + usize = s2u(size); malloc_mutex_lock(&arena->lock); - run = arena_run_alloc_large(arena, size, zero); + run = arena_run_alloc_large(arena, usize, zero); if (run == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); @@ -1535,15 +1504,17 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) miscelm = arena_run_to_miscelm(run); ret = arena_miscelm_to_rpages(miscelm); if (config_stats) { + index_t index = size2index(usize) - NBINS; + arena->stats.nmalloc_large++; arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; - arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; - arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; + arena->stats.allocated_large += usize; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; } if (config_prof) - idump = arena_prof_accum_locked(arena, size); + idump = arena_prof_accum_locked(arena, usize); malloc_mutex_unlock(&arena->lock); if (config_prof && idump) prof_idump(); @@ -1551,9 +1522,9 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) if (!zero) { if (config_fill) { if (unlikely(opt_junk)) - memset(ret, 0xa5, size); + memset(ret, 0xa5, usize); else if (unlikely(opt_zero)) - memset(ret, 0, size); + memset(ret, 0, usize); } } @@ -1610,12 +1581,14 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) ret = arena_miscelm_to_rpages(miscelm); if (config_stats) { + index_t index = size2index(size) - NBINS; + arena->stats.nmalloc_large++; arena->stats.nrequests_large++; arena->stats.allocated_large += size; - arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; - arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; - arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; } malloc_mutex_unlock(&arena->lock); @@ -1632,22 +1605,23 @@ void arena_prof_promoted(const void *ptr, size_t size) { arena_chunk_t *chunk; - size_t pageind, binind; + size_t pageind; + index_t binind; cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - assert(isalloc(ptr, false) == PAGE); - assert(isalloc(ptr, true) == PAGE); + assert(isalloc(ptr, false) == LARGE_MINCLASS); + assert(isalloc(ptr, true) == LARGE_MINCLASS); assert(size <= SMALL_MAXCLASS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - binind = small_size2bin(size); + binind = size2index(size); assert(binind < NBINS); arena_mapbits_large_binind_set(chunk, pageind, binind); - assert(isalloc(ptr, false) == PAGE); + assert(isalloc(ptr, false) == LARGE_MINCLASS); assert(isalloc(ptr, true) == size); } @@ -1660,7 +1634,7 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, if (run == bin->runcur) bin->runcur = NULL; else { - size_t binind = arena_bin_index(chunk->arena, bin); + index_t binind = arena_bin_index(chunk->arena, bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; if (bin_info->nregs != 1) { @@ -1678,7 +1652,7 @@ static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin) { - size_t binind; + index_t binind; arena_bin_info_t *bin_info; size_t npages, run_ind, past; arena_chunk_map_misc_t *miscelm; @@ -1762,7 +1736,8 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_run_t *run; arena_bin_t *bin; arena_bin_info_t *bin_info; - size_t size, binind; + size_t size; + index_t binind; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); @@ -1851,10 +1826,12 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) arena_dalloc_junk_large(ptr, usize); if (config_stats) { + index_t index = size2index(usize) - NBINS; + arena->stats.ndalloc_large++; arena->stats.allocated_large -= usize; - arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; - arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; + arena->stats.lstats[index].ndalloc++; + arena->stats.lstats[index].curruns--; } } @@ -1887,17 +1864,20 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, malloc_mutex_lock(&arena->lock); arena_run_trim_tail(arena, chunk, run, oldsize, size, true); if (config_stats) { + index_t oldindex = size2index(oldsize) - NBINS; + index_t index = size2index(size) - NBINS; + arena->stats.ndalloc_large++; arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; + arena->stats.lstats[oldindex].ndalloc++; + arena->stats.lstats[oldindex].curruns--; arena->stats.nmalloc_large++; arena->stats.nrequests_large++; arena->stats.allocated_large += size; - arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; - arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; - arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; } malloc_mutex_unlock(&arena->lock); } @@ -1909,24 +1889,30 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; size_t npages = oldsize >> LG_PAGE; size_t followsize; + size_t usize_min = s2u(size); assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); /* Try to extend the run. */ - assert(size + extra > oldsize); + assert(usize_min > oldsize); malloc_mutex_lock(&arena->lock); if (pageind + npages < chunk_npages && arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && (followsize = arena_mapbits_unallocated_size_get(chunk, - pageind+npages)) >= size - oldsize) { + pageind+npages)) >= usize_min - oldsize) { /* * The next run is available and sufficiently large. Split the * following run, then merge the first part with the existing * allocation. */ - size_t flag_dirty; - size_t splitsize = (oldsize + followsize <= size + extra) - ? followsize : size + extra - oldsize; + size_t flag_dirty, splitsize, usize; + + usize = s2u(size + extra); + while (oldsize + followsize < usize) + usize = index2size(size2index(usize)-1); + assert(usize >= usize_min); + splitsize = usize - oldsize; + arena_run_t *run = &arena_miscelm_get(chunk, pageind+npages)->run; arena_run_split_large(arena, run, splitsize, zero); @@ -1948,17 +1934,20 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); if (config_stats) { + index_t oldindex = size2index(oldsize) - NBINS; + index_t index = size2index(size) - NBINS; + arena->stats.ndalloc_large++; arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; + arena->stats.lstats[oldindex].ndalloc++; + arena->stats.lstats[oldindex].curruns--; arena->stats.nmalloc_large++; arena->stats.nrequests_large++; arena->stats.allocated_large += size; - arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; - arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; - arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; + arena->stats.lstats[index].nmalloc++; + arena->stats.lstats[index].nrequests++; + arena->stats.lstats[index].curruns++; } malloc_mutex_unlock(&arena->lock); return (false); @@ -1996,10 +1985,14 @@ static bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { - size_t psize; + size_t usize; - psize = PAGE_CEILING(size + extra); - if (psize == oldsize) { + /* Make sure extra can't cause size_t overflow. */ + if (extra >= arena_maxclass) + return (true); + + usize = s2u(size + extra); + if (usize == oldsize) { /* Same size class. */ return (false); } else { @@ -2009,16 +2002,15 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena = chunk->arena; - if (psize < oldsize) { + if (usize < oldsize) { /* Fill before shrinking in order avoid a race. */ - arena_ralloc_junk_large(ptr, oldsize, psize); + arena_ralloc_junk_large(ptr, oldsize, usize); arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, - psize); + usize); return (false); } else { bool ret = arena_ralloc_large_grow(arena, chunk, ptr, - oldsize, PAGE_CEILING(size), - psize - PAGE_CEILING(size), zero); + oldsize, size, extra, zero); if (config_fill && !ret && !zero) { if (unlikely(opt_junk)) { memset((void *)((uintptr_t)ptr + @@ -2045,12 +2037,11 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, */ if (oldsize <= arena_maxclass) { if (oldsize <= SMALL_MAXCLASS) { - assert(arena_bin_info[small_size2bin(oldsize)].reg_size + assert(arena_bin_info[size2index(oldsize)].reg_size == oldsize); - if ((size + extra <= SMALL_MAXCLASS && - small_size2bin(size + extra) == - small_size2bin(oldsize)) || (size <= oldsize && - size + extra >= oldsize)) + if ((size + extra <= SMALL_MAXCLASS && size2index(size + + extra) == size2index(oldsize)) || (size <= oldsize + && size + extra >= oldsize)) return (false); } else { assert(size <= arena_maxclass); @@ -2258,7 +2249,7 @@ arena_new(arena_t *arena, unsigned ind) /* * Calculate bin_info->run_size such that it meets the following constraints: * - * *) bin_info->run_size <= arena_maxclass + * *) bin_info->run_size <= arena_maxrun * *) bin_info->nregs <= RUN_MAXREGS * * bin_info->nregs and bin_info->reg0_offset are also calculated here, since @@ -2330,7 +2321,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info) /* * Make sure that the run will fit within an arena chunk. */ - while (actual_run_size > arena_maxclass) { + while (actual_run_size > arena_maxrun) { actual_run_size -= PAGE; actual_nregs = (actual_run_size - pad_size) / bin_info->reg_interval; @@ -2396,7 +2387,17 @@ arena_boot(void) map_misc_offset = offsetof(arena_chunk_t, map_bits) + sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); - arena_maxclass = chunksize - (map_bias << LG_PAGE); + arena_maxrun = chunksize - (map_bias << LG_PAGE); + arena_maxclass = index2size(size2index(chunksize)-1); + if (arena_maxclass > arena_maxrun) { + /* + * For small chunk sizes it's possible for there to be fewer + * non-header pages available than are necessary to serve the + * size classes just below chunksize. + */ + arena_maxclass = arena_maxrun; + } + nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS); bin_info_init(); } diff --git a/src/chunk.c b/src/chunk.c index 32b8b3a6..618aaca0 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -27,9 +27,6 @@ rtree_t *chunks_rtree; size_t chunksize; size_t chunksize_mask; /* (chunksize - 1). */ size_t chunk_npages; -size_t map_bias; -size_t map_misc_offset; -size_t arena_maxclass; /* Max size class for arenas. */ /******************************************************************************/ /* diff --git a/src/ctl.c b/src/ctl.c index 309f1f65..f1f3234b 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1628,7 +1628,7 @@ arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) } CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t) -CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t) +CTL_RO_NL_GEN(arenas_lrun_i_size, index2size(NBINS+mib[2]), size_t) static const ctl_named_node_t * arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) { diff --git a/src/huge.c b/src/huge.c index 6bdc0767..ae416253 100644 --- a/src/huge.c +++ b/src/huge.c @@ -15,12 +15,19 @@ static extent_tree_t huge; void * huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero) { + size_t usize; - return (huge_palloc(tsd, arena, size, chunksize, zero)); + usize = s2u(size); + if (usize == 0) { + /* size_t overflow. */ + return (NULL); + } + + return (huge_palloc(tsd, arena, usize, chunksize, zero)); } void * -huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, +huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, bool zero) { void *ret; @@ -30,11 +37,8 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, /* Allocate one or more contiguous chunks for this request. */ - csize = CHUNK_CEILING(size); - if (csize == 0) { - /* size is large enough to cause size_t wrap-around. */ - return (NULL); - } + csize = CHUNK_CEILING(usize); + assert(csize >= usize); /* Allocate an extent node with which to track the chunk. */ node = base_node_alloc(); @@ -55,7 +59,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, /* Insert node into huge. */ node->addr = ret; - node->size = csize; + node->size = usize; node->arena = arena; malloc_mutex_lock(&huge_mtx); @@ -64,9 +68,9 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, if (config_fill && !zero) { if (unlikely(opt_junk)) - memset(ret, 0xa5, csize); + memset(ret, 0xa5, usize); else if (unlikely(opt_zero) && !is_zeroed) - memset(ret, 0, csize); + memset(ret, 0, usize); } return (ret); @@ -97,7 +101,7 @@ huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); static bool huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { - size_t csize; + size_t usize; void *expand_addr; size_t expand_size; extent_node_t *node, key; @@ -105,14 +109,14 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { bool is_zeroed; void *ret; - csize = CHUNK_CEILING(size); - if (csize == 0) { - /* size is large enough to cause size_t wrap-around. */ + usize = s2u(size); + if (usize == 0) { + /* size_t overflow. */ return (true); } - expand_addr = ptr + oldsize; - expand_size = csize - oldsize; + expand_addr = ptr + CHUNK_CEILING(oldsize); + expand_size = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); malloc_mutex_lock(&huge_mtx); @@ -140,14 +144,14 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { malloc_mutex_lock(&huge_mtx); /* Update the size of the huge allocation. */ - node->size = csize; + node->size = usize; malloc_mutex_unlock(&huge_mtx); if (config_fill && !zero) { if (unlikely(opt_junk)) - memset(expand_addr, 0xa5, expand_size); + memset(ptr + oldsize, 0xa5, usize - oldsize); else if (unlikely(opt_zero) && !is_zeroed) - memset(expand_addr, 0, expand_size); + memset(ptr + oldsize, 0, usize - oldsize); } return (false); } @@ -156,27 +160,71 @@ bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { + size_t usize; /* Both allocations must be huge to avoid a move. */ - if (oldsize <= arena_maxclass) + if (oldsize < chunksize) return (true); - assert(CHUNK_CEILING(oldsize) == oldsize); + assert(s2u(oldsize) == oldsize); + usize = s2u(size); + if (usize == 0) { + /* size_t overflow. */ + return (true); + } /* - * Avoid moving the allocation if the size class can be left the same. + * Avoid moving the allocation if the existing chunk size accommodates + * the new size. */ + if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize) + && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { + size_t usize_next; + + /* Increase usize to incorporate extra. */ + while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < + oldsize) + usize = usize_next; + + /* Update the size of the huge allocation if it changed. */ + if (oldsize != usize) { + extent_node_t *node, key; + + malloc_mutex_lock(&huge_mtx); + + key.addr = ptr; + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + + assert(node->size != usize); + node->size = usize; + + malloc_mutex_unlock(&huge_mtx); + + if (oldsize < usize) { + if (zero || (config_fill && + unlikely(opt_zero))) { + memset(ptr + oldsize, 0, usize - + oldsize); + } else if (config_fill && unlikely(opt_junk)) { + memset(ptr + oldsize, 0xa5, usize - + oldsize); + } + } else if (config_fill && unlikely(opt_junk) && oldsize + > usize) + memset(ptr + usize, 0x5a, oldsize - usize); + } + return (false); + } + if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { return (false); } - /* Overflow. */ - if (CHUNK_CEILING(size) == 0) - return (true); - /* Shrink the allocation in-place. */ - if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(size)) { + if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize)) { extent_node_t *node, key; void *excess_addr; size_t excess_size; @@ -189,15 +237,15 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, assert(node->addr == ptr); /* Update the size of the huge allocation. */ - node->size = CHUNK_CEILING(size); + node->size = usize; malloc_mutex_unlock(&huge_mtx); - excess_addr = node->addr + CHUNK_CEILING(size); - excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(size); + excess_addr = node->addr + CHUNK_CEILING(usize); + excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); /* Zap the excess chunks. */ - huge_dalloc_junk(excess_addr, excess_size); + huge_dalloc_junk(ptr + usize, oldsize - usize); arena_chunk_dalloc_huge(node->arena, excess_addr, excess_size); return (false); @@ -275,7 +323,8 @@ huge_dalloc(void *ptr) malloc_mutex_unlock(&huge_mtx); huge_dalloc_junk(node->addr, node->size); - arena_chunk_dalloc_huge(node->arena, node->addr, node->size); + arena_chunk_dalloc_huge(node->arena, node->addr, + CHUNK_CEILING(node->size)); base_node_dalloc(node); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 3490ecdf..f3750b40 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -42,6 +42,38 @@ unsigned narenas_auto; /* Set to true once the allocator has been initialized. */ static bool malloc_initialized = false; +JEMALLOC_ALIGNED(CACHELINE) +const size_t index2size_tab[NSIZES] = { +#define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ + ((ZU(1)<next_gc_bin; + index_t binind = tcache->next_gc_bin; tcache_bin_t *tbin = &tcache->tbins[binind]; tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; @@ -62,7 +62,7 @@ tcache_event_hard(tcache_t *tcache) } void * -tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) +tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, index_t binind) { void *ret; @@ -76,7 +76,7 @@ tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) } void -tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, +tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, tcache_t *tcache) { void *ptr; @@ -153,7 +153,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, } void -tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, +tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, tcache_t *tcache) { void *ptr; diff --git a/test/unit/junk.c b/test/unit/junk.c index 301428f2..5b35a879 100644 --- a/test/unit/junk.c +++ b/test/unit/junk.c @@ -88,7 +88,6 @@ test_junk(size_t sz_min, size_t sz_max) if (xallocx(s, sz+1, 0, 0) == sz) { void *junked = (void *)s; - s = (char *)rallocx(s, sz+1, 0); assert_ptr_not_null((void *)s, "Unexpected rallocx() failure"); @@ -134,13 +133,25 @@ TEST_END arena_ralloc_junk_large_t *arena_ralloc_junk_large_orig; static void *most_recently_trimmed; +static size_t +shrink_size(size_t size) +{ + size_t shrink_size; + + for (shrink_size = size - 1; nallocx(shrink_size, 0) == size; + shrink_size--) + ; /* Do nothing. */ + + return (shrink_size); +} + static void arena_ralloc_junk_large_intercept(void *ptr, size_t old_usize, size_t usize) { arena_ralloc_junk_large_orig(ptr, old_usize, usize); assert_zu_eq(old_usize, arena_maxclass, "Unexpected old_usize"); - assert_zu_eq(usize, arena_maxclass-PAGE, "Unexpected usize"); + assert_zu_eq(usize, shrink_size(arena_maxclass), "Unexpected usize"); most_recently_trimmed = ptr; } @@ -154,7 +165,7 @@ TEST_BEGIN(test_junk_large_ralloc_shrink) arena_ralloc_junk_large_orig = arena_ralloc_junk_large; arena_ralloc_junk_large = arena_ralloc_junk_large_intercept; - p2 = rallocx(p1, arena_maxclass-PAGE, 0); + p2 = rallocx(p1, shrink_size(arena_maxclass), 0); assert_ptr_eq(p1, p2, "Unexpected move during shrink"); arena_ralloc_junk_large = arena_ralloc_junk_large_orig; diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index c70473cc..e62e54f2 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -357,7 +357,7 @@ TEST_BEGIN(test_arenas_lrun_constants) assert_zu_eq(name, expected, "Incorrect "#name" size"); \ } while (0) - TEST_ARENAS_LRUN_CONSTANT(size_t, size, (1 << LG_PAGE)); + TEST_ARENAS_LRUN_CONSTANT(size_t, size, (1 << (LG_PAGE+2))); #undef TEST_ARENAS_LRUN_CONSTANT } From bf40641c5c9496d2912ad9ff2c38ee9ce2bfbde6 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 6 Oct 2014 16:35:11 -0700 Subject: [PATCH 141/352] Fix a prof_tctx_t destruction race. --- src/prof.c | 50 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/src/prof.c b/src/prof.c index a6cea92f..b3150a27 100644 --- a/src/prof.c +++ b/src/prof.c @@ -609,7 +609,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) { prof_tdata_t *tdata = tctx->tdata; prof_gctx_t *gctx = tctx->gctx; - bool destroy_tdata, destroy_gctx; + bool destroy_tdata, destroy_tctx, destroy_gctx; assert(tctx->cnts.curobjs == 0); assert(tctx->cnts.curbytes == 0); @@ -622,25 +622,38 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) malloc_mutex_unlock(tdata->lock); malloc_mutex_lock(gctx->lock); - tctx_tree_remove(&gctx->tctxs, tctx); - if (prof_gctx_should_destroy(gctx)) { + if (tctx->state != prof_tctx_state_dumping) { + tctx_tree_remove(&gctx->tctxs, tctx); + destroy_tctx = true; + if (prof_gctx_should_destroy(gctx)) { + /* + * Increment gctx->nlimbo in order to keep another + * thread from winning the race to destroy gctx while + * this one has gctx->lock dropped. Without this, it + * would be possible for another thread to: + * + * 1) Sample an allocation associated with gctx. + * 2) Deallocate the sampled object. + * 3) Successfully prof_gctx_try_destroy(gctx). + * + * The result would be that gctx no longer exists by the + * time this thread accesses it in + * prof_gctx_try_destroy(). + */ + gctx->nlimbo++; + destroy_gctx = true; + } else + destroy_gctx = false; + } else { /* - * Increment gctx->nlimbo in order to keep another thread from - * winning the race to destroy gctx while this one has - * gctx->lock dropped. Without this, it would be possible for - * another thread to: - * - * 1) Sample an allocation associated with gctx. - * 2) Deallocate the sampled object. - * 3) Successfully prof_gctx_try_destroy(gctx). - * - * The result would be that gctx no longer exists by the time - * this thread accesses it in prof_gctx_try_destroy(). + * A dumping thread needs tctx to remain valid until dumping + * has finished. Change state such that the dumping thread will + * complete destruction during a late dump iteration phase. */ - gctx->nlimbo++; - destroy_gctx = true; - } else + tctx->state = prof_tctx_state_purgatory; + destroy_tctx = false; destroy_gctx = false; + } malloc_mutex_unlock(gctx->lock); if (destroy_gctx) prof_gctx_try_destroy(tsd, gctx, tdata); @@ -648,7 +661,8 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) if (destroy_tdata) prof_tdata_destroy(tsd, tdata, false); - idalloc(tsd, tctx); + if (destroy_tctx) + idalloc(tsd, tctx); } static bool From 8bb3198f72fc7587dc93527f9f19fb5be52fa553 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 7 Oct 2014 23:14:57 -0700 Subject: [PATCH 142/352] Refactor/fix arenas manipulation. Abstract arenas access to use arena_get() (or a0get() where appropriate) rather than directly reading e.g. arenas[ind]. Prior to the addition of the arenas.extend mallctl, the worst possible outcome of directly accessing arenas was a stale read, but arenas.extend may allocate and assign a new array to arenas. Add a tsd-based arenas_cache, which amortizes arenas reads. This introduces some subtle bootstrapping issues, with tsd_boot() now being split into tsd_boot[01]() to support tsd wrapper allocation bootstrapping, as well as an arenas_cache_bypass tsd variable which dynamically terminates allocation of arenas_cache itself. Promote a0malloc(), a0calloc(), and a0free() to be generally useful for internal allocation, and use them in several places (more may be appropriate). Abstract arena->nthreads management and fix a missing decrement during thread destruction (recent tsd refactoring left arenas_cleanup() unused). Change arena_choose() to propagate OOM, and handle OOM in all callers. This is important for providing consistent allocation behavior when the MALLOCX_ARENA() flag is being used. Prior to this fix, it was possible for an OOM to result in allocation silently allocating from a different arena than the one specified. --- include/jemalloc/internal/arena.h | 14 +- .../jemalloc/internal/jemalloc_internal.h.in | 90 +-- include/jemalloc/internal/private_symbols.txt | 28 +- include/jemalloc/internal/tcache.h | 1 + include/jemalloc/internal/tsd.h | 239 ++++++-- src/arena.c | 30 +- src/chunk.c | 10 +- src/ctl.c | 119 ++-- src/huge.c | 6 +- src/jemalloc.c | 526 +++++++++++++----- src/tcache.c | 14 +- src/tsd.c | 19 +- test/unit/tsd.c | 1 + 13 files changed, 745 insertions(+), 352 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 681b5802..894ce9af 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -389,7 +389,7 @@ bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats); -bool arena_new(arena_t *arena, unsigned ind); +arena_t *arena_new(unsigned ind); void arena_boot(void); void arena_prefork(arena_t *arena); void arena_postfork_parent(arena_t *arena); @@ -924,8 +924,10 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, true)) != NULL)) return (tcache_alloc_small(tcache, size, zero)); else { - return (arena_malloc_small(choose_arena(tsd, arena), - size, zero)); + arena = arena_choose(tsd, arena); + if (unlikely(arena == NULL)) + return (NULL); + return (arena_malloc_small(arena, size, zero)); } } else { /* @@ -936,8 +938,10 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, tcache_get(tsd, true)) != NULL)) return (tcache_alloc_large(tcache, size, zero)); else { - return (arena_malloc_large(choose_arena(tsd, arena), - size, zero)); + arena = arena_choose(tsd, arena); + if (unlikely(arena == NULL)) + return (NULL); + return (arena_malloc_large(arena, size, zero)); } } } diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 8f0beb9e..c7a5fd8a 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -386,20 +386,6 @@ extern bool in_valgrind; /* Number of CPUs. */ extern unsigned ncpus; -/* Protects arenas initialization (arenas, arenas_total). */ -extern malloc_mutex_t arenas_lock; -/* - * Arenas that are used to service external requests. Not all elements of the - * arenas array are necessarily used; arenas are created lazily as needed. - * - * arenas[0..narenas_auto) are used for automatic multiplexing of threads and - * arenas. arenas[narenas_auto..narenas_total) are only used if the application - * takes some action to create them and allocate from them. - */ -extern arena_t **arenas; -extern unsigned narenas_total; -extern unsigned narenas_auto; /* Read-only after initialization. */ - /* * index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by index2size_compute(). @@ -412,11 +398,23 @@ extern size_t const index2size_tab[NSIZES]; */ extern uint8_t const size2index_tab[]; +arena_t *a0get(void); +void *a0malloc(size_t size); +void *a0calloc(size_t num, size_t size); +void a0free(void *ptr); arena_t *arenas_extend(unsigned ind); -arena_t *choose_arena_hard(tsd_t *tsd); +arena_t *arena_init(unsigned ind); +unsigned narenas_total_get(void); +arena_t *arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing); +arena_t *arena_choose_hard(tsd_t *tsd); +void arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind); +unsigned arena_nbound(unsigned ind); void thread_allocated_cleanup(tsd_t *tsd); void thread_deallocated_cleanup(tsd_t *tsd); void arena_cleanup(tsd_t *tsd); +void arenas_cache_cleanup(tsd_t *tsd); +void narenas_cache_cleanup(tsd_t *tsd); +void arenas_cache_bypass_cleanup(tsd_t *tsd); void jemalloc_prefork(void); void jemalloc_postfork_parent(void); void jemalloc_postfork_child(void); @@ -475,8 +473,9 @@ size_t s2u_compute(size_t size); size_t s2u_lookup(size_t size); size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment); -unsigned narenas_total_get(void); -arena_t *choose_arena(tsd_t *tsd, arena_t *arena); +arena_t *arena_choose(tsd_t *tsd, arena_t *arena); +arena_t *arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, + bool refresh_if_missing); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) @@ -709,34 +708,51 @@ sa2u(size_t size, size_t alignment) return (usize); } -JEMALLOC_INLINE unsigned -narenas_total_get(void) -{ - unsigned narenas; - - malloc_mutex_lock(&arenas_lock); - narenas = narenas_total; - malloc_mutex_unlock(&arenas_lock); - - return (narenas); -} - /* Choose an arena based on a per-thread value. */ JEMALLOC_INLINE arena_t * -choose_arena(tsd_t *tsd, arena_t *arena) +arena_choose(tsd_t *tsd, arena_t *arena) { arena_t *ret; if (arena != NULL) return (arena); - if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) { - ret = choose_arena_hard(tsd); - assert(ret != NULL); - } + if (unlikely((ret = tsd_arena_get(tsd)) == NULL)) + ret = arena_choose_hard(tsd); return (ret); } + +JEMALLOC_INLINE arena_t * +arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, + bool refresh_if_missing) +{ + arena_t *arena; + arena_t **arenas_cache = tsd_arenas_cache_get(tsd); + + /* init_if_missing requires refresh_if_missing. */ + assert(!init_if_missing || refresh_if_missing); + + if (unlikely(arenas_cache == NULL)) { + /* arenas_cache hasn't been initialized yet. */ + return (arena_get_hard(tsd, ind, init_if_missing)); + } + if (unlikely(ind >= tsd_narenas_cache_get(tsd))) { + /* + * ind is invalid, cache is old (too small), or arena to be + * initialized. + */ + return (refresh_if_missing ? arena_get_hard(tsd, ind, + init_if_missing) : NULL); + } + arena = arenas_cache[ind]; + if (likely(arena != NULL) || !refresh_if_missing) + return (arena); + if (init_if_missing) + return (arena_get_hard(tsd, ind, init_if_missing)); + else + return (NULL); +} #endif #include "jemalloc/internal/bitmap.h" @@ -833,8 +849,10 @@ ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, ret = arena_malloc(tsd, arena, usize, zero, try_tcache); else { if (usize <= arena_maxclass) { - ret = arena_palloc(choose_arena(tsd, arena), usize, - alignment, zero); + arena = arena_choose(tsd, arena); + if (unlikely(arena == NULL)) + return (NULL); + ret = arena_palloc(arena, usize, alignment, zero); } else if (alignment <= chunksize) ret = huge_malloc(tsd, arena, usize, zero); else diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 1a7fde4b..d5e6fdcf 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -1,11 +1,16 @@ a0calloc a0free +a0get a0malloc +arena_get +arena_get_hard arena_alloc_junk_small arena_bin_index arena_bin_info arena_bitselm_get arena_boot +arena_choose +arena_choose_hard arena_chunk_alloc_huge arena_chunk_dalloc_huge arena_cleanup @@ -19,6 +24,7 @@ arena_dalloc_large_locked arena_dalloc_small arena_dss_prec_get arena_dss_prec_set +arena_init arena_malloc arena_malloc_large arena_malloc_small @@ -42,9 +48,11 @@ arena_mapbitsp_read arena_mapbitsp_write arena_maxclass arena_maxrun +arena_migrate arena_miscelm_get arena_miscelm_to_pageind arena_miscelm_to_rpages +arena_nbound arena_new arena_palloc arena_postfork_child @@ -69,10 +77,8 @@ arena_salloc arena_sdalloc arena_stats_merge arena_tcache_fill_small -arenas -arenas_cleanup -arenas_extend -arenas_lock +arenas_cache_bypass_cleanup +arenas_cache_cleanup atomic_add_u atomic_add_uint32 atomic_add_uint64 @@ -100,8 +106,6 @@ bitmap_size bitmap_unset bt_init buferror -choose_arena -choose_arena_hard chunk_alloc_arena chunk_alloc_base chunk_alloc_default @@ -247,7 +251,8 @@ malloc_mutex_unlock malloc_printf malloc_snprintf malloc_strtoumax -malloc_tsd_boot +malloc_tsd_boot0 +malloc_tsd_boot1 malloc_tsd_cleanup_register malloc_tsd_dalloc malloc_tsd_malloc @@ -259,8 +264,7 @@ map_bias map_misc_offset mb_write mutex_boot -narenas_auto -narenas_total +narenas_cache_cleanup narenas_total_get ncpus nhbins @@ -363,6 +367,7 @@ tcache_alloc_small tcache_alloc_small_hard tcache_arena_associate tcache_arena_dissociate +tcache_arena_reassociate tcache_bin_flush_large tcache_bin_flush_small tcache_bin_info @@ -388,11 +393,14 @@ tsd_booted tsd_arena_get tsd_arena_set tsd_boot +tsd_boot0 +tsd_boot1 tsd_cleanup tsd_cleanup_wrapper tsd_fetch tsd_get -tsd_get_wrapper +tsd_wrapper_get +tsd_wrapper_set tsd_initialized tsd_init_check_recursion tsd_init_finish diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index da8e4ef4..02eec5db 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -109,6 +109,7 @@ void tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, void tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_associate(tcache_t *tcache, arena_t *arena); +void tcache_arena_reassociate(tcache_t *tcache, arena_t *arena); void tcache_arena_dissociate(tcache_t *tcache); tcache_t *tcache_get_hard(tsd_t *tsd); tcache_t *tcache_create(arena_t *arena); diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 25450391..b5658f8e 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -2,7 +2,7 @@ #ifdef JEMALLOC_H_TYPES /* Maximum number of malloc_tsd users with cleanup functions. */ -#define MALLOC_TSD_CLEANUPS_MAX 8 +#define MALLOC_TSD_CLEANUPS_MAX 2 typedef bool (*malloc_tsd_cleanup_t)(void); @@ -23,7 +23,7 @@ typedef enum { /* * TLS/TSD-agnostic macro-based implementation of thread-specific data. There - * are four macros that support (at least) three use cases: file-private, + * are five macros that support (at least) three use cases: file-private, * library-private, and library-private inlined. Following is an example * library-private tsd variable: * @@ -33,18 +33,19 @@ typedef enum { * int y; * } example_t; * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) - * malloc_tsd_protos(, example_, example_t *) - * malloc_tsd_externs(example_, example_t *) + * malloc_tsd_types(example_, example_t) + * malloc_tsd_protos(, example_, example_t) + * malloc_tsd_externs(example_, example_t) * In example.c: - * malloc_tsd_data(, example_, example_t *, EX_INITIALIZER) - * malloc_tsd_funcs(, example_, example_t *, EX_INITIALIZER, + * malloc_tsd_data(, example_, example_t, EX_INITIALIZER) + * malloc_tsd_funcs(, example_, example_t, EX_INITIALIZER, * example_tsd_cleanup) * * The result is a set of generated functions, e.g.: * * bool example_tsd_boot(void) {...} - * example_t **example_tsd_get() {...} - * void example_tsd_set(example_t **val) {...} + * example_t *example_tsd_get() {...} + * void example_tsd_set(example_t *val) {...} * * Note that all of the functions deal in terms of (a_type *) rather than * (a_type) so that it is possible to support non-pointer types (unlike @@ -70,9 +71,32 @@ typedef enum { * non-NULL. */ +/* malloc_tsd_types(). */ +#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP +#define malloc_tsd_types(a_name, a_type) +#elif (defined(JEMALLOC_TLS)) +#define malloc_tsd_types(a_name, a_type) +#elif (defined(_WIN32)) +#define malloc_tsd_types(a_name, a_type) \ +typedef struct { \ + bool initialized; \ + a_type val; \ +} a_name##tsd_wrapper_t; +#else +#define malloc_tsd_types(a_name, a_type) \ +typedef struct { \ + bool initialized; \ + a_type val; \ +} a_name##tsd_wrapper_t; +#endif + /* malloc_tsd_protos(). */ #define malloc_tsd_protos(a_attr, a_name, a_type) \ a_attr bool \ +a_name##tsd_boot0(void); \ +a_attr void \ +a_name##tsd_boot1(void); \ +a_attr bool \ a_name##tsd_boot(void); \ a_attr a_type * \ a_name##tsd_get(void); \ @@ -93,11 +117,13 @@ extern bool a_name##tsd_booted; #elif (defined(_WIN32)) #define malloc_tsd_externs(a_name, a_type) \ extern DWORD a_name##tsd_tsd; \ +extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ extern bool a_name##tsd_booted; #else #define malloc_tsd_externs(a_name, a_type) \ extern pthread_key_t a_name##tsd_tsd; \ extern tsd_init_head_t a_name##tsd_init_head; \ +extern a_name##tsd_wrapper_t a_name##tsd_boot_wrapper; \ extern bool a_name##tsd_booted; #endif @@ -118,6 +144,10 @@ a_attr bool a_name##tsd_booted = false; #elif (defined(_WIN32)) #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ a_attr DWORD a_name##tsd_tsd; \ +a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ + false, \ + a_initializer \ +}; \ a_attr bool a_name##tsd_booted = false; #else #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ @@ -126,6 +156,10 @@ a_attr tsd_init_head_t a_name##tsd_init_head = { \ ql_head_initializer(blocks), \ MALLOC_MUTEX_INITIALIZER \ }; \ +a_attr a_name##tsd_wrapper_t a_name##tsd_boot_wrapper = { \ + false, \ + a_initializer \ +}; \ a_attr bool a_name##tsd_booted = false; #endif @@ -145,7 +179,7 @@ a_name##tsd_cleanup_wrapper(void) \ return (a_name##tsd_initialized); \ } \ a_attr bool \ -a_name##tsd_boot(void) \ +a_name##tsd_boot0(void) \ { \ \ if (a_cleanup != malloc_tsd_no_cleanup) { \ @@ -155,6 +189,18 @@ a_name##tsd_boot(void) \ a_name##tsd_booted = true; \ return (false); \ } \ +a_attr void \ +a_name##tsd_boot1() \ +{ \ + \ + /* Do nothing. */ \ +} \ +a_attr bool \ +a_name##tsd_boot(void) \ +{ \ + \ + return (a_name##tsd_boot0()); \ +} \ /* Get/set. */ \ a_attr a_type * \ a_name##tsd_get(void) \ @@ -177,7 +223,7 @@ a_name##tsd_set(a_type *val) \ a_cleanup) \ /* Initialization/cleanup. */ \ a_attr bool \ -a_name##tsd_boot(void) \ +a_name##tsd_boot0(void) \ { \ \ if (a_cleanup != malloc_tsd_no_cleanup) { \ @@ -188,6 +234,18 @@ a_name##tsd_boot(void) \ a_name##tsd_booted = true; \ return (false); \ } \ +a_attr void \ +a_name##tsd_boot1() \ +{ \ + \ + /* Do nothing. */ \ +} \ +a_attr bool \ +a_name##tsd_boot(void) \ +{ \ + \ + return (a_name##tsd_boot0()); \ +} \ /* Get/set. */ \ a_attr a_type * \ a_name##tsd_get(void) \ @@ -215,11 +273,6 @@ a_name##tsd_set(a_type *val) \ #elif (defined(_WIN32)) #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ a_cleanup) \ -/* Data structure. */ \ -typedef struct { \ - bool initialized; \ - a_type val; \ -} a_name##tsd_wrapper_t; \ /* Initialization/cleanup. */ \ a_attr bool \ a_name##tsd_cleanup_wrapper(void) \ @@ -241,23 +294,18 @@ a_name##tsd_cleanup_wrapper(void) \ malloc_tsd_dalloc(wrapper); \ return (false); \ } \ -a_attr bool \ -a_name##tsd_boot(void) \ +a_attr void \ +a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ { \ \ - a_name##tsd_tsd = TlsAlloc(); \ - if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ - return (true); \ - if (a_cleanup != malloc_tsd_no_cleanup) { \ - malloc_tsd_cleanup_register( \ - &a_name##tsd_cleanup_wrapper); \ + if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ + malloc_write(": Error setting" \ + " TSD for "#a_name"\n"); \ + abort(); \ } \ - a_name##tsd_booted = true; \ - return (false); \ } \ -/* Get/set. */ \ a_attr a_name##tsd_wrapper_t * \ -a_name##tsd_get_wrapper(void) \ +a_name##tsd_wrapper_get(void) \ { \ a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ TlsGetValue(a_name##tsd_tsd); \ @@ -273,21 +321,63 @@ a_name##tsd_get_wrapper(void) \ wrapper->initialized = false; \ wrapper->val = a_initializer; \ } \ - if (!TlsSetValue(a_name##tsd_tsd, (void *)wrapper)) { \ - malloc_write(": Error setting" \ - " TSD for "#a_name"\n"); \ - abort(); \ - } \ + a_name##tsd_wrapper_set(wrapper); \ } \ return (wrapper); \ } \ +a_attr bool \ +a_name##tsd_boot0(void) \ +{ \ + \ + a_name##tsd_tsd = TlsAlloc(); \ + if (a_name##tsd_tsd == TLS_OUT_OF_INDEXES) \ + return (true); \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + malloc_tsd_cleanup_register( \ + &a_name##tsd_cleanup_wrapper); \ + } \ + a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ + a_name##tsd_booted = true; \ + return (false); \ +} \ +a_attr void \ +a_name##tsd_boot1() \ +{ \ + a_name##tsd_wrapper_t *wrapper; \ + wrapper = (a_name##tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ + if (wrapper == NULL) { \ + malloc_write(": Error allocating" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } \ + memcpy(wrapper, &a_name##tsd_boot_wrapper, \ + sizeof(a_name##tsd_wrapper_t)); \ + a_name##tsd_wrapper_set(wrapper); \ +} \ +a_attr bool \ +a_name##tsd_boot(void) \ +{ \ + \ + if (a_name##tsd_boot0()) \ + return (true); \ + a_name##tsd_boot1(); \ + return (false); \ +} \ +a_attr bool \ +a_name##tsd_boot(void) \ +{ \ + \ + return (false); \ +} \ +/* Get/set. */ \ a_attr a_type * \ a_name##tsd_get(void) \ { \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_get_wrapper(); \ + wrapper = a_name##tsd_wrapper_get(); \ return (&wrapper->val); \ } \ a_attr void \ @@ -296,7 +386,7 @@ a_name##tsd_set(a_type *val) \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_get_wrapper(); \ + wrapper = a_name##tsd_wrapper_get(); \ wrapper->val = *(val); \ if (a_cleanup != malloc_tsd_no_cleanup) \ wrapper->initialized = true; \ @@ -304,11 +394,6 @@ a_name##tsd_set(a_type *val) \ #else #define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ a_cleanup) \ -/* Data structure. */ \ -typedef struct { \ - bool initialized; \ - a_type val; \ -} a_name##tsd_wrapper_t; \ /* Initialization/cleanup. */ \ a_attr void \ a_name##tsd_cleanup_wrapper(void *arg) \ @@ -333,19 +418,19 @@ a_name##tsd_cleanup_wrapper(void *arg) \ } \ malloc_tsd_dalloc(wrapper); \ } \ -a_attr bool \ -a_name##tsd_boot(void) \ +a_attr void \ +a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ { \ \ - if (pthread_key_create(&a_name##tsd_tsd, \ - a_name##tsd_cleanup_wrapper) != 0) \ - return (true); \ - a_name##tsd_booted = true; \ - return (false); \ + if (pthread_setspecific(a_name##tsd_tsd, \ + (void *)wrapper)) { \ + malloc_write(": Error setting" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } \ } \ -/* Get/set. */ \ a_attr a_name##tsd_wrapper_t * \ -a_name##tsd_get_wrapper(void) \ +a_name##tsd_wrapper_get(void) \ { \ a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ pthread_getspecific(a_name##tsd_tsd); \ @@ -367,23 +452,54 @@ a_name##tsd_get_wrapper(void) \ wrapper->initialized = false; \ wrapper->val = a_initializer; \ } \ - if (pthread_setspecific(a_name##tsd_tsd, \ - (void *)wrapper)) { \ - malloc_write(": Error setting" \ - " TSD for "#a_name"\n"); \ - abort(); \ - } \ + a_name##tsd_wrapper_set(wrapper); \ tsd_init_finish(&a_name##tsd_init_head, &block); \ } \ return (wrapper); \ } \ +a_attr bool \ +a_name##tsd_boot0(void) \ +{ \ + \ + if (pthread_key_create(&a_name##tsd_tsd, \ + a_name##tsd_cleanup_wrapper) != 0) \ + return (true); \ + a_name##tsd_wrapper_set(&a_name##tsd_boot_wrapper); \ + a_name##tsd_booted = true; \ + return (false); \ +} \ +a_attr void \ +a_name##tsd_boot1() \ +{ \ + a_name##tsd_wrapper_t *wrapper; \ + wrapper = (a_name##tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##tsd_wrapper_t)); \ + if (wrapper == NULL) { \ + malloc_write(": Error allocating" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } \ + memcpy(wrapper, &a_name##tsd_boot_wrapper, \ + sizeof(a_name##tsd_wrapper_t)); \ + a_name##tsd_wrapper_set(wrapper); \ +} \ +a_attr bool \ +a_name##tsd_boot(void) \ +{ \ + \ + if (a_name##tsd_boot0()) \ + return (true); \ + a_name##tsd_boot1(); \ + return (false); \ +} \ +/* Get/set. */ \ a_attr a_type * \ a_name##tsd_get(void) \ { \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_get_wrapper(); \ + wrapper = a_name##tsd_wrapper_get(); \ return (&wrapper->val); \ } \ a_attr void \ @@ -392,7 +508,7 @@ a_name##tsd_set(a_type *val) \ a_name##tsd_wrapper_t *wrapper; \ \ assert(a_name##tsd_booted); \ - wrapper = a_name##tsd_get_wrapper(); \ + wrapper = a_name##tsd_wrapper_get(); \ wrapper->val = *(val); \ if (a_cleanup != malloc_tsd_no_cleanup) \ wrapper->initialized = true; \ @@ -423,6 +539,9 @@ struct tsd_init_head_s { O(thread_deallocated, uint64_t) \ O(prof_tdata, prof_tdata_t *) \ O(arena, arena_t *) \ + O(arenas_cache, arena_t **) \ + O(narenas_cache, unsigned) \ + O(arenas_cache_bypass, bool) \ O(tcache_enabled, tcache_enabled_t) \ O(quarantine, quarantine_t *) \ @@ -433,6 +552,9 @@ struct tsd_init_head_s { 0, \ NULL, \ NULL, \ + NULL, \ + 0, \ + false, \ tcache_enabled_default, \ NULL \ } @@ -447,6 +569,8 @@ MALLOC_TSD static const tsd_t tsd_initializer = TSD_INITIALIZER; +malloc_tsd_types(, tsd_t) + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS @@ -455,7 +579,8 @@ void *malloc_tsd_malloc(size_t size); void malloc_tsd_dalloc(void *wrapper); void malloc_tsd_no_cleanup(void *arg); void malloc_tsd_cleanup_register(bool (*f)(void)); -bool malloc_tsd_boot(void); +bool malloc_tsd_boot0(void); +void malloc_tsd_boot1(void); #if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ !defined(_WIN32)) void *tsd_init_check_recursion(tsd_init_head_t *head, diff --git a/src/arena.c b/src/arena.c index 49a30572..86e54404 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2192,27 +2192,37 @@ arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, } } -bool -arena_new(arena_t *arena, unsigned ind) +arena_t * +arena_new(unsigned ind) { + arena_t *arena; unsigned i; arena_bin_t *bin; + /* + * Allocate arena and arena->lstats contiguously, mainly because there + * is no way to clean up if base_alloc() OOMs. + */ + if (config_stats) { + arena = (arena_t *)base_alloc(CACHELINE_CEILING(sizeof(arena_t)) + + nlclasses * sizeof(malloc_large_stats_t)); + } else + arena = (arena_t *)base_alloc(sizeof(arena_t)); + if (arena == NULL) + return (NULL); + arena->ind = ind; arena->nthreads = 0; arena->chunk_alloc = chunk_alloc_default; arena->chunk_dalloc = chunk_dalloc_default; if (malloc_mutex_init(&arena->lock)) - return (true); + return (NULL); if (config_stats) { memset(&arena->stats, 0, sizeof(arena_stats_t)); - arena->stats.lstats = - (malloc_large_stats_t *)base_alloc(nlclasses * - sizeof(malloc_large_stats_t)); - if (arena->stats.lstats == NULL) - return (true); + arena->stats.lstats = (malloc_large_stats_t *)(((void *)arena) + + CACHELINE_CEILING(sizeof(arena_t))); memset(arena->stats.lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); if (config_tcache) @@ -2236,14 +2246,14 @@ arena_new(arena_t *arena, unsigned ind) for (i = 0; i < NBINS; i++) { bin = &arena->bins[i]; if (malloc_mutex_init(&bin->lock)) - return (true); + return (NULL); bin->runcur = NULL; arena_run_tree_new(&bin->runs); if (config_stats) memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } - return (false); + return (arena); } /* diff --git a/src/chunk.c b/src/chunk.c index 618aaca0..f65b67af 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -254,9 +254,17 @@ void * chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind) { + arena_t *arena; + + arena = arena_get(tsd_fetch(), arena_ind, false, true); + /* + * The arena we're allocating on behalf of must have been initialized + * already. + */ + assert(arena != NULL); return (chunk_alloc_core(new_addr, size, alignment, false, zero, - arenas[arena_ind]->dss_prec)); + arena->dss_prec)); } static void diff --git a/src/ctl.c b/src/ctl.c index f1f3234b..37f8f42a 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -447,7 +447,7 @@ ctl_arena_init(ctl_arena_stats_t *astats) { if (astats->lstats == NULL) { - astats->lstats = (malloc_large_stats_t *)base_alloc(nlclasses * + astats->lstats = (malloc_large_stats_t *)a0malloc(nlclasses * sizeof(malloc_large_stats_t)); if (astats->lstats == NULL) return (true); @@ -567,31 +567,24 @@ ctl_arena_refresh(arena_t *arena, unsigned i) static bool ctl_grow(void) { - tsd_t *tsd; ctl_arena_stats_t *astats; - arena_t **tarenas; - tsd = tsd_fetch(); + /* Initialize new arena. */ + if (arena_init(ctl_stats.narenas) == NULL) + return (true); - /* Allocate extended arena stats and arenas arrays. */ - astats = (ctl_arena_stats_t *)imalloc(tsd, (ctl_stats.narenas + 2) * + /* Allocate extended arena stats. */ + astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t)); if (astats == NULL) return (true); - tarenas = (arena_t **)imalloc(tsd, (ctl_stats.narenas + 1) * - sizeof(arena_t *)); - if (tarenas == NULL) { - idalloc(tsd, astats); - return (true); - } /* Initialize the new astats element. */ memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) { - idalloc(tsd, tarenas); - idalloc(tsd, astats); + a0free(astats); return (true); } /* Swap merged stats to their new location. */ @@ -604,32 +597,7 @@ ctl_grow(void) memcpy(&astats[ctl_stats.narenas + 1], &tstats, sizeof(ctl_arena_stats_t)); } - /* Initialize the new arenas element. */ - tarenas[ctl_stats.narenas] = NULL; - { - arena_t **arenas_old = arenas; - /* - * Swap extended arenas array into place. Although ctl_mtx - * protects this function from other threads extending the - * array, it does not protect from other threads mutating it - * (i.e. initializing arenas and setting array elements to - * point to them). Therefore, array copying must happen under - * the protection of arenas_lock. - */ - malloc_mutex_lock(&arenas_lock); - arenas = tarenas; - memcpy(arenas, arenas_old, ctl_stats.narenas * - sizeof(arena_t *)); - narenas_total++; - arenas_extend(narenas_total - 1); - malloc_mutex_unlock(&arenas_lock); - /* - * Deallocate arenas_old only if it came from imalloc() (not - * base_alloc()). - */ - if (ctl_stats.narenas != narenas_auto) - idalloc(tsd, arenas_old); - } + a0free(ctl_stats.arenas); ctl_stats.arenas = astats; ctl_stats.narenas++; @@ -639,6 +607,7 @@ ctl_grow(void) static void ctl_refresh(void) { + tsd_t *tsd; unsigned i; VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); @@ -657,15 +626,17 @@ ctl_refresh(void) ctl_stats.arenas[ctl_stats.narenas].nthreads = 0; ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]); - malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas); + tsd = tsd_fetch(); + for (i = 0; i < ctl_stats.narenas; i++) + tarenas[i] = arena_get(tsd, i, false, (i == 0)); + for (i = 0; i < ctl_stats.narenas; i++) { - if (arenas[i] != NULL) - ctl_stats.arenas[i].nthreads = arenas[i]->nthreads; + if (tarenas[i] != NULL) + ctl_stats.arenas[i].nthreads = arena_nbound(i); else ctl_stats.arenas[i].nthreads = 0; } - malloc_mutex_unlock(&arenas_lock); + for (i = 0; i < ctl_stats.narenas; i++) { bool initialized = (tarenas[i] != NULL); @@ -698,9 +669,8 @@ ctl_init(void) * Allocate space for one extra arena stats element, which * contains summed stats across all arenas. */ - assert(narenas_auto == narenas_total_get()); - ctl_stats.narenas = narenas_auto; - ctl_stats.arenas = (ctl_arena_stats_t *)base_alloc( + ctl_stats.narenas = narenas_total_get(); + ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc( (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); if (ctl_stats.arenas == NULL) { ret = true; @@ -718,6 +688,13 @@ ctl_init(void) unsigned i; for (i = 0; i <= ctl_stats.narenas; i++) { if (ctl_arena_init(&ctl_stats.arenas[i])) { + unsigned j; + for (j = 0; j < i; j++) { + a0free( + ctl_stats.arenas[j].lstats); + } + a0free(ctl_stats.arenas); + ctl_stats.arenas = NULL; ret = true; goto label_return; } @@ -1231,17 +1208,19 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, { int ret; tsd_t *tsd; + arena_t *arena; unsigned newind, oldind; tsd = tsd_fetch(); + arena = arena_choose(tsd, NULL); + if (arena == NULL) + return (EAGAIN); malloc_mutex_lock(&ctl_mtx); - newind = oldind = choose_arena(tsd, NULL)->ind; + newind = oldind = arena->ind; WRITE(newind, unsigned); READ(oldind, unsigned); if (newind != oldind) { - arena_t *arena; - if (newind >= ctl_stats.narenas) { /* New arena index is out of range. */ ret = EFAULT; @@ -1249,28 +1228,18 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, } /* Initialize arena if necessary. */ - malloc_mutex_lock(&arenas_lock); - if ((arena = arenas[newind]) == NULL && (arena = - arenas_extend(newind)) == NULL) { - malloc_mutex_unlock(&arenas_lock); + arena = arena_get(tsd, newind, true, true); + if (arena == NULL) { ret = EAGAIN; goto label_return; } - assert(arena == arenas[newind]); - arenas[oldind]->nthreads--; - arenas[newind]->nthreads++; - malloc_mutex_unlock(&arenas_lock); - - /* Set new arena association. */ + /* Set new arena/tcache associations. */ + arena_migrate(tsd, oldind, newind); if (config_tcache) { tcache_t *tcache = tsd_tcache_get(tsd); - if (tcache != NULL) { - tcache_arena_dissociate(tcache); - tcache_arena_associate(tcache, arena); - } + if (tcache != NULL) + tcache_arena_reassociate(tcache, arena); } - - tsd_arena_set(tsd, arena); } ret = 0; @@ -1400,11 +1369,13 @@ label_return: static void arena_purge(unsigned arena_ind) { + tsd_t *tsd; + unsigned i; VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); - malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas); - malloc_mutex_unlock(&arenas_lock); + tsd = tsd_fetch(); + for (i = 0; i < ctl_stats.narenas; i++) + tarenas[i] = arena_get(tsd, i, false, (i == 0)); if (arena_ind == ctl_stats.narenas) { unsigned i; @@ -1467,7 +1438,7 @@ arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, } if (arena_ind < ctl_stats.narenas) { - arena_t *arena = arenas[arena_ind]; + arena_t *arena = arena_get(tsd_fetch(), arena_ind, false, true); if (arena == NULL || (dss_prec != dss_prec_limit && arena_dss_prec_set(arena, dss_prec))) { ret = EFAULT; @@ -1501,7 +1472,8 @@ arena_i_chunk_alloc_ctl(const size_t *mib, size_t miblen, void *oldp, arena_t *arena; malloc_mutex_lock(&ctl_mtx); - if (arena_ind < narenas_total && (arena = arenas[arena_ind]) != NULL) { + if (arena_ind < narenas_total_get() && (arena = arena_get(tsd_fetch(), + arena_ind, false, true)) != NULL) { malloc_mutex_lock(&arena->lock); READ(arena->chunk_alloc, chunk_alloc_t *); WRITE(arena->chunk_alloc, chunk_alloc_t *); @@ -1527,7 +1499,8 @@ arena_i_chunk_dalloc_ctl(const size_t *mib, size_t miblen, void *oldp, arena_t *arena; malloc_mutex_lock(&ctl_mtx); - if (arena_ind < narenas_total && (arena = arenas[arena_ind]) != NULL) { + if (arena_ind < narenas_total_get() && (arena = arena_get(tsd_fetch(), + arena_ind, false, true)) != NULL) { malloc_mutex_lock(&arena->lock); READ(arena->chunk_dalloc, chunk_dalloc_t *); WRITE(arena->chunk_dalloc, chunk_dalloc_t *); diff --git a/src/huge.c b/src/huge.c index ae416253..1376729a 100644 --- a/src/huge.c +++ b/src/huge.c @@ -50,7 +50,11 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, * it is possible to make correct junk/zero fill decisions below. */ is_zeroed = zero; - arena = choose_arena(tsd, arena); + arena = arena_choose(tsd, arena); + if (unlikely(arena == NULL)) { + base_node_dalloc(node); + return (NULL); + } ret = arena_chunk_alloc_huge(arena, NULL, csize, alignment, &is_zeroed); if (ret == NULL) { base_node_dalloc(node); diff --git a/src/jemalloc.c b/src/jemalloc.c index f3750b40..3c889e8a 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -4,8 +4,6 @@ /******************************************************************************/ /* Data. */ -malloc_tsd_data(, arenas, arena_t *, NULL) - /* Runtime configuration options. */ const char *je_malloc_conf JEMALLOC_ATTR(weak); bool opt_abort = @@ -34,10 +32,20 @@ bool in_valgrind; unsigned ncpus; -malloc_mutex_t arenas_lock; -arena_t **arenas; -unsigned narenas_total; -unsigned narenas_auto; +/* Protects arenas initialization (arenas, narenas_total). */ +static malloc_mutex_t arenas_lock; +/* + * Arenas that are used to service external requests. Not all elements of the + * arenas array are necessarily used; arenas are created lazily as needed. + * + * arenas[0..narenas_auto) are used for automatic multiplexing of threads and + * arenas. arenas[narenas_auto..narenas_total) are only used if the application + * takes some action to create them and allocate from them. + */ +static arena_t **arenas; +static unsigned narenas_total; +static arena_t *a0; /* arenas[0]; read-only after initialization. */ +static unsigned narenas_auto; /* Read-only after initialization. */ /* Set to true once the allocator has been initialized. */ static bool malloc_initialized = false; @@ -144,35 +152,288 @@ static bool malloc_init_hard(void); * Begin miscellaneous support functions. */ -/* Create a new arena and insert it into the arenas array at index ind. */ -arena_t * -arenas_extend(unsigned ind) +JEMALLOC_ALWAYS_INLINE_C void +malloc_thread_init(void) { - arena_t *ret; - - ret = (arena_t *)base_alloc(sizeof(arena_t)); - if (ret != NULL && !arena_new(ret, ind)) { - arenas[ind] = ret; - return (ret); - } - /* Only reached if there is an OOM error. */ /* - * OOM here is quite inconvenient to propagate, since dealing with it - * would require a check for failure in the fast path. Instead, punt - * by using arenas[0]. In practice, this is an extremely unlikely - * failure. + * TSD initialization can't be safely done as a side effect of + * deallocation, because it is possible for a thread to do nothing but + * deallocate its TLS data via free(), in which case writing to TLS + * would cause write-after-free memory corruption. The quarantine + * facility *only* gets used as a side effect of deallocation, so make + * a best effort attempt at initializing its TSD by hooking all + * allocation events. */ - malloc_write(": Error initializing arena\n"); - if (opt_abort) - abort(); - - return (arenas[0]); + if (config_fill && unlikely(opt_quarantine)) + quarantine_alloc_hook(); } -/* Slow path, called only by choose_arena(). */ +JEMALLOC_ALWAYS_INLINE_C bool +malloc_init(void) +{ + + if (unlikely(!malloc_initialized) && malloc_init_hard()) + return (true); + malloc_thread_init(); + + return (false); +} + +/* + * The a0*() functions are used instead of i[mcd]alloc() in bootstrap-sensitive + * situations that cannot tolerate TLS variable access. These functions are + * also exposed for use in static binaries on FreeBSD, hence the old-style + * malloc() API. + */ + arena_t * -choose_arena_hard(tsd_t *tsd) +a0get(void) +{ + + assert(a0 != NULL); + return (a0); +} + +static void * +a0alloc(size_t size, bool zero) +{ + void *ret; + + if (unlikely(malloc_init())) + return (NULL); + + if (size == 0) + size = 1; + + if (size <= arena_maxclass) + ret = arena_malloc(NULL, a0get(), size, zero, false); + else + ret = huge_malloc(NULL, a0get(), size, zero); + + return (ret); +} + +void * +a0malloc(size_t size) +{ + + return (a0alloc(size, false)); +} + +void * +a0calloc(size_t num, size_t size) +{ + + return (a0alloc(num * size, true)); +} + +void +a0free(void *ptr) +{ + arena_chunk_t *chunk; + + if (ptr == NULL) + return; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) + arena_dalloc(NULL, chunk, ptr, false); + else + huge_dalloc(ptr); +} + +/* Create a new arena and insert it into the arenas array at index ind. */ +arena_t * +arena_init(unsigned ind) +{ + arena_t *arena; + + malloc_mutex_lock(&arenas_lock); + + /* Expand arenas if necessary. */ + assert(ind <= narenas_total); + if (ind == narenas_total) { + unsigned narenas_new = narenas_total + 1; + arena_t **arenas_new = + (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * + sizeof(arena_t *))); + if (arenas_new == NULL) { + arena = NULL; + goto label_return; + } + memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); + arenas_new[ind] = NULL; + /* + * Deallocate only if arenas came from a0malloc() (not + * base_alloc()). + */ + if (narenas_total != narenas_auto) + a0free(arenas); + arenas = arenas_new; + narenas_total = narenas_new; + } + + /* + * Another thread may have already initialized arenas[ind] if it's an + * auto arena. + */ + arena = arenas[ind]; + if (arena != NULL) { + assert(ind < narenas_auto); + goto label_return; + } + + /* Actually initialize the arena. */ + arena = arenas[ind] = arena_new(ind); +label_return: + malloc_mutex_unlock(&arenas_lock); + return (arena); +} + +unsigned +narenas_total_get(void) +{ + unsigned narenas; + + malloc_mutex_lock(&arenas_lock); + narenas = narenas_total; + malloc_mutex_unlock(&arenas_lock); + + return (narenas); +} + +static void +arena_bind_locked(tsd_t *tsd, unsigned ind) +{ + arena_t *arena; + + arena = arenas[ind]; + arena->nthreads++; + + if (tsd_nominal(tsd)) + tsd_arena_set(tsd, arena); +} + +static void +arena_bind(tsd_t *tsd, unsigned ind) +{ + + malloc_mutex_lock(&arenas_lock); + arena_bind_locked(tsd, ind); + malloc_mutex_unlock(&arenas_lock); +} + +void +arena_migrate(tsd_t *tsd, unsigned oldind, unsigned newind) +{ + arena_t *oldarena, *newarena; + + malloc_mutex_lock(&arenas_lock); + oldarena = arenas[oldind]; + newarena = arenas[newind]; + oldarena->nthreads--; + newarena->nthreads++; + malloc_mutex_unlock(&arenas_lock); + tsd_arena_set(tsd, newarena); +} + +unsigned +arena_nbound(unsigned ind) +{ + unsigned nthreads; + + malloc_mutex_lock(&arenas_lock); + nthreads = arenas[ind]->nthreads; + malloc_mutex_unlock(&arenas_lock); + return (nthreads); +} + +static void +arena_unbind(tsd_t *tsd, unsigned ind) +{ + arena_t *arena; + + malloc_mutex_lock(&arenas_lock); + arena = arenas[ind]; + arena->nthreads--; + malloc_mutex_unlock(&arenas_lock); + tsd_arena_set(tsd, NULL); +} + +arena_t * +arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) +{ + arena_t *arena; + arena_t **arenas_cache = tsd_arenas_cache_get(tsd); + unsigned narenas_cache = tsd_narenas_cache_get(tsd); + unsigned narenas_actual = narenas_total_get(); + + /* Deallocate old cache if it's too small. */ + if (arenas_cache != NULL && narenas_cache < narenas_actual) { + a0free(arenas_cache); + arenas_cache = NULL; + narenas_cache = 0; + tsd_arenas_cache_set(tsd, arenas_cache); + tsd_narenas_cache_set(tsd, narenas_cache); + } + + /* Allocate cache if it's missing. */ + if (arenas_cache == NULL) { + bool *arenas_cache_bypassp = tsd_arenas_cache_bypassp_get(tsd); + assert(ind < narenas_actual || !init_if_missing); + narenas_cache = (ind < narenas_actual) ? narenas_actual : ind+1; + + if (!*arenas_cache_bypassp) { + *arenas_cache_bypassp = true; + arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * + narenas_cache); + *arenas_cache_bypassp = false; + } else + arenas_cache = NULL; + if (arenas_cache == NULL) { + /* + * This function must always tell the truth, even if + * it's slow, so don't let OOM or recursive allocation + * avoidance (note arenas_cache_bypass check) get in the + * way. + */ + if (ind >= narenas_actual) + return (NULL); + malloc_mutex_lock(&arenas_lock); + arena = arenas[ind]; + malloc_mutex_unlock(&arenas_lock); + return (arena); + } + tsd_arenas_cache_set(tsd, arenas_cache); + tsd_narenas_cache_set(tsd, narenas_cache); + } + + /* + * Copy to cache. It's possible that the actual number of arenas has + * increased since narenas_total_get() was called above, but that causes + * no correctness issues unless two threads concurrently execute the + * arenas.extend mallctl, which we trust mallctl synchronization to + * prevent. + */ + malloc_mutex_lock(&arenas_lock); + memcpy(arenas_cache, arenas, sizeof(arena_t *) * narenas_actual); + malloc_mutex_unlock(&arenas_lock); + if (narenas_cache > narenas_actual) { + memset(&arenas_cache[narenas_actual], 0, sizeof(arena_t *) * + (narenas_cache - narenas_actual)); + } + + /* Read the refreshed cache, and init the arena if necessary. */ + arena = arenas_cache[ind]; + if (init_if_missing && arena == NULL) + arena = arenas_cache[ind] = arena_init(ind); + return (arena); +} + +/* Slow path, called only by arena_choose(). */ +arena_t * +arena_choose_hard(tsd_t *tsd) { arena_t *ret; @@ -182,7 +443,7 @@ choose_arena_hard(tsd_t *tsd) choose = 0; first_null = narenas_auto; malloc_mutex_lock(&arenas_lock); - assert(arenas[0] != NULL); + assert(a0get() != NULL); for (i = 1; i < narenas_auto; i++) { if (arenas[i] != NULL) { /* @@ -215,20 +476,20 @@ choose_arena_hard(tsd_t *tsd) ret = arenas[choose]; } else { /* Initialize a new arena. */ - ret = arenas_extend(first_null); + choose = first_null; + ret = arena_init(choose); + if (ret == NULL) { + malloc_mutex_unlock(&arenas_lock); + return (NULL); + } } - ret->nthreads++; + arena_bind_locked(tsd, choose); malloc_mutex_unlock(&arenas_lock); } else { - ret = arenas[0]; - malloc_mutex_lock(&arenas_lock); - ret->nthreads++; - malloc_mutex_unlock(&arenas_lock); + ret = a0get(); + arena_bind(tsd, 0); } - if (tsd_nominal(tsd)) - tsd_arena_set(tsd, ret); - return (ret); } @@ -248,6 +509,33 @@ thread_deallocated_cleanup(tsd_t *tsd) void arena_cleanup(tsd_t *tsd) +{ + arena_t *arena; + + arena = tsd_arena_get(tsd); + if (arena != NULL) + arena_unbind(tsd, arena->ind); +} + +void +arenas_cache_cleanup(tsd_t *tsd) +{ + arena_t **arenas_cache; + + arenas_cache = tsd_arenas_cache_get(tsd); + if (arenas != NULL) + a0free(arenas_cache); +} + +void +narenas_cache_cleanup(tsd_t *tsd) +{ + + /* Do nothing. */ +} + +void +arenas_cache_bypass_cleanup(tsd_t *tsd) { /* Do nothing. */ @@ -312,44 +600,6 @@ malloc_ncpus(void) return ((result == -1) ? 1 : (unsigned)result); } -void -arenas_cleanup(void *arg) -{ - arena_t *arena = *(arena_t **)arg; - - malloc_mutex_lock(&arenas_lock); - arena->nthreads--; - malloc_mutex_unlock(&arenas_lock); -} - -JEMALLOC_ALWAYS_INLINE_C void -malloc_thread_init(void) -{ - - /* - * TSD initialization can't be safely done as a side effect of - * deallocation, because it is possible for a thread to do nothing but - * deallocate its TLS data via free(), in which case writing to TLS - * would cause write-after-free memory corruption. The quarantine - * facility *only* gets used as a side effect of deallocation, so make - * a best effort attempt at initializing its TSD by hooking all - * allocation events. - */ - if (config_fill && unlikely(opt_quarantine)) - quarantine_alloc_hook(); -} - -JEMALLOC_ALWAYS_INLINE_C bool -malloc_init(void) -{ - - if (unlikely(!malloc_initialized) && malloc_init_hard()) - return (true); - malloc_thread_init(); - - return (false); -} - static bool malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, char const **v_p, size_t *vlen_p) @@ -745,7 +995,7 @@ malloc_init_hard(void) #endif malloc_initializer = INITIALIZER; - if (malloc_tsd_boot()) { + if (malloc_tsd_boot0()) { malloc_mutex_unlock(&init_lock); return (true); } @@ -809,10 +1059,10 @@ malloc_init_hard(void) /* * Initialize one arena here. The rest are lazily created in - * choose_arena_hard(). + * arena_choose_hard(). */ - arenas_extend(0); - if (arenas[0] == NULL) { + a0 = arena_init(0); + if (a0 == NULL) { malloc_mutex_unlock(&init_lock); return (true); } @@ -887,6 +1137,7 @@ malloc_init_hard(void) malloc_initialized = true; malloc_mutex_unlock(&init_lock); + malloc_tsd_boot1(); return (false); } @@ -1428,8 +1679,8 @@ JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = * Begin non-standard functions. */ -JEMALLOC_ALWAYS_INLINE_C void -imallocx_flags_decode_hard(size_t size, int flags, size_t *usize, +JEMALLOC_ALWAYS_INLINE_C bool +imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize, size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) { @@ -1444,16 +1695,19 @@ imallocx_flags_decode_hard(size_t size, int flags, size_t *usize, if ((flags & MALLOCX_ARENA_MASK) != 0) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); *try_tcache = false; - *arena = arenas[arena_ind]; + *arena = arena_get(tsd, arena_ind, true, true); + if (unlikely(*arena == NULL)) + return (true); } else { *try_tcache = true; *arena = NULL; } + return (false); } -JEMALLOC_ALWAYS_INLINE_C void -imallocx_flags_decode(size_t size, int flags, size_t *usize, size_t *alignment, - bool *zero, bool *try_tcache, arena_t **arena) +JEMALLOC_ALWAYS_INLINE_C bool +imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, + size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) { if (likely(flags == 0)) { @@ -1463,9 +1717,10 @@ imallocx_flags_decode(size_t size, int flags, size_t *usize, size_t *alignment, *zero = false; *try_tcache = true; *arena = NULL; + return (false); } else { - imallocx_flags_decode_hard(size, flags, usize, alignment, zero, - try_tcache, arena); + return (imallocx_flags_decode_hard(tsd, size, flags, usize, + alignment, zero, try_tcache, arena)); } } @@ -1524,8 +1779,9 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) arena_t *arena; prof_tctx_t *tctx; - imallocx_flags_decode(size, flags, usize, &alignment, &zero, - &try_tcache, &arena); + if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, + &zero, &try_tcache, &arena))) + return (NULL); tctx = prof_alloc_prep(tsd, *usize, true); if (likely((uintptr_t)tctx == (uintptr_t)1U)) { p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment, @@ -1558,8 +1814,9 @@ imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) return (imalloc(tsd, size)); } - imallocx_flags_decode_hard(size, flags, usize, &alignment, &zero, - &try_tcache, &arena); + if (unlikely(imallocx_flags_decode_hard(tsd, size, flags, usize, + &alignment, &zero, &try_tcache, &arena))) + return (NULL); return (imallocx_flags(tsd, *usize, alignment, zero, try_tcache, arena)); } @@ -1685,9 +1942,10 @@ je_rallocx(void *ptr, size_t size, int flags) arena_chunk_t *chunk; try_tcache_alloc = false; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - try_tcache_dalloc = (chunk == ptr || chunk->arena != - arenas[arena_ind]); - arena = arenas[arena_ind]; + arena = arena_get(tsd, arena_ind, true, true); + if (unlikely(arena == NULL)) + goto label_oom; + try_tcache_dalloc = (chunk == ptr || chunk->arena != arena); } else { try_tcache_alloc = true; try_tcache_dalloc = true; @@ -1825,6 +2083,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); + // XX Dangerous arenas read. arena = arenas[arena_ind]; } else arena = NULL; @@ -1875,16 +2134,24 @@ je_sallocx(const void *ptr, int flags) void je_dallocx(void *ptr, int flags) { + tsd_t *tsd; bool try_tcache; assert(ptr != NULL); assert(malloc_initialized || IS_INITIALIZER); + tsd = tsd_fetch(); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - try_tcache = (chunk == ptr || chunk->arena != - arenas[arena_ind]); + arena_t *arena = arena_get(tsd, arena_ind, true, true); + /* + * If arena is NULL, the application passed an arena that has + * never been used before, which is unsupported during + * deallocation. + */ + assert(arena != NULL); + try_tcache = (chunk == ptr || chunk->arena != arena); } else try_tcache = true; @@ -1908,6 +2175,7 @@ inallocx(size_t size, int flags) void je_sdallocx(void *ptr, size_t size, int flags) { + tsd_t *tsd; bool try_tcache; size_t usize; @@ -1916,16 +2184,22 @@ je_sdallocx(void *ptr, size_t size, int flags) usize = inallocx(size, flags); assert(usize == isalloc(ptr, config_prof)); + tsd = tsd_fetch(); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - try_tcache = (chunk == ptr || chunk->arena != - arenas[arena_ind]); + arena_t *arena = arena_get(tsd, arena_ind, true, true); + /* + * If arena is NULL, the application passed an arena that has + * never been used before, which is unsupported during + * deallocation. + */ + try_tcache = (chunk == ptr || chunk->arena != arena); } else try_tcache = true; UTRACE(ptr, 0, 0); - isfree(tsd_fetch(), ptr, usize, try_tcache); + isfree(tsd, ptr, usize, try_tcache); } size_t @@ -2105,55 +2379,3 @@ jemalloc_postfork_child(void) } /******************************************************************************/ -/* - * The following functions are used for TLS allocation/deallocation in static - * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() - * is that these avoid accessing TLS variables. - */ - -static void * -a0alloc(size_t size, bool zero) -{ - - if (unlikely(malloc_init())) - return (NULL); - - if (size == 0) - size = 1; - - if (size <= arena_maxclass) - return (arena_malloc(NULL, arenas[0], size, zero, false)); - else - return (huge_malloc(NULL, arenas[0], size, zero)); -} - -void * -a0malloc(size_t size) -{ - - return (a0alloc(size, false)); -} - -void * -a0calloc(size_t num, size_t size) -{ - - return (a0alloc(num * size, true)); -} - -void -a0free(void *ptr) -{ - arena_chunk_t *chunk; - - if (ptr == NULL) - return; - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) - arena_dalloc(NULL, chunk, ptr, false); - else - huge_dalloc(ptr); -} - -/******************************************************************************/ diff --git a/src/tcache.c b/src/tcache.c index 2c968c68..1bf70269 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -245,6 +245,14 @@ tcache_arena_associate(tcache_t *tcache, arena_t *arena) tcache->arena = arena; } +void +tcache_arena_reassociate(tcache_t *tcache, arena_t *arena) +{ + + tcache_arena_dissociate(tcache); + tcache_arena_associate(tcache, arena); +} + void tcache_arena_dissociate(tcache_t *tcache) { @@ -261,13 +269,17 @@ tcache_arena_dissociate(tcache_t *tcache) tcache_t * tcache_get_hard(tsd_t *tsd) { + arena_t *arena; if (!tcache_enabled_get()) { if (tsd_nominal(tsd)) tcache_enabled_set(false); /* Memoize. */ return (NULL); } - return (tcache_create(choose_arena(tsd, NULL))); + arena = arena_choose(tsd, NULL); + if (unlikely(arena == NULL)) + return (NULL); + return (tcache_create(arena)); } tcache_t * diff --git a/src/tsd.c b/src/tsd.c index cbc64e44..59253fe3 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -15,16 +15,14 @@ void * malloc_tsd_malloc(size_t size) { - /* Avoid choose_arena() in order to dodge bootstrapping issues. */ - return (arena_malloc(NULL, arenas[0], CACHELINE_CEILING(size), false, - false)); + return (a0malloc(CACHELINE_CEILING(size))); } void malloc_tsd_dalloc(void *wrapper) { - idalloct(NULL, wrapper, false); + a0free(wrapper); } void @@ -106,15 +104,24 @@ MALLOC_TSD } bool -malloc_tsd_boot(void) +malloc_tsd_boot0(void) { ncleanups = 0; - if (tsd_boot()) + if (tsd_boot0()) return (true); + *tsd_arenas_cache_bypassp_get(tsd_fetch()) = true; return (false); } +void +malloc_tsd_boot1(void) +{ + + tsd_boot1(); + *tsd_arenas_cache_bypassp_get(tsd_fetch()) = false; +} + #ifdef _WIN32 static BOOL WINAPI _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) diff --git a/test/unit/tsd.c b/test/unit/tsd.c index eb1c5976..b031c484 100644 --- a/test/unit/tsd.c +++ b/test/unit/tsd.c @@ -6,6 +6,7 @@ typedef unsigned int data_t; static bool data_cleanup_executed; +malloc_tsd_types(data_, data_t) malloc_tsd_protos(, data_, data_t) void From f22214a29ddd3bed005cbcc8f2aff7c61ef4940b Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Mon, 6 Oct 2014 03:42:10 -0400 Subject: [PATCH 143/352] Use regular arena allocation for huge tree nodes. This avoids grabbing the base mutex, as a step towards fine-grained locking for huge allocations. The thread cache also provides a tiny (~3%) improvement for serial huge allocations. --- include/jemalloc/internal/huge.h | 2 +- .../jemalloc/internal/jemalloc_internal.h.in | 4 +-- src/huge.c | 9 ++++--- src/jemalloc.c | 2 +- test/unit/junk.c | 27 ++++++++++++++----- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 939993f2..5d4d3a16 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -21,7 +21,7 @@ void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; #endif -void huge_dalloc(void *ptr); +void huge_dalloc(tsd_t *tsd, void *ptr); size_t huge_salloc(const void *ptr); prof_tctx_t *huge_prof_tctx_get(const void *ptr); void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index c7a5fd8a..f4d5de6a 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -938,7 +938,7 @@ idalloct(tsd_t *tsd, void *ptr, bool try_tcache) if (chunk != ptr) arena_dalloc(tsd, chunk, ptr, try_tcache); else - huge_dalloc(ptr); + huge_dalloc(tsd, ptr); } JEMALLOC_ALWAYS_INLINE void @@ -952,7 +952,7 @@ isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) if (chunk != ptr) arena_sdalloc(tsd, chunk, ptr, size, try_tcache); else - huge_dalloc(ptr); + huge_dalloc(tsd, ptr); } JEMALLOC_ALWAYS_INLINE void diff --git a/src/huge.c b/src/huge.c index 1376729a..541df60a 100644 --- a/src/huge.c +++ b/src/huge.c @@ -41,7 +41,8 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, assert(csize >= usize); /* Allocate an extent node with which to track the chunk. */ - node = base_node_alloc(); + node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), + CACHELINE, false, tsd != NULL, NULL); if (node == NULL) return (NULL); @@ -57,7 +58,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, } ret = arena_chunk_alloc_huge(arena, NULL, csize, alignment, &is_zeroed); if (ret == NULL) { - base_node_dalloc(node); + idalloct(tsd, node, tsd != NULL); return (NULL); } @@ -311,7 +312,7 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, } void -huge_dalloc(void *ptr) +huge_dalloc(tsd_t *tsd, void *ptr) { extent_node_t *node, key; @@ -329,7 +330,7 @@ huge_dalloc(void *ptr) huge_dalloc_junk(node->addr, node->size); arena_chunk_dalloc_huge(node->arena, node->addr, CHUNK_CEILING(node->size)); - base_node_dalloc(node); + idalloct(tsd, node, tsd != NULL); } size_t diff --git a/src/jemalloc.c b/src/jemalloc.c index 3c889e8a..38b5aaf7 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -240,7 +240,7 @@ a0free(void *ptr) if (chunk != ptr) arena_dalloc(NULL, chunk, ptr, false); else - huge_dalloc(ptr); + huge_dalloc(NULL, ptr); } /* Create a new arena and insert it into the arenas array at index ind. */ diff --git a/test/unit/junk.c b/test/unit/junk.c index 5b35a879..1522a610 100644 --- a/test/unit/junk.c +++ b/test/unit/junk.c @@ -8,7 +8,16 @@ const char *malloc_conf = static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig; static arena_dalloc_junk_large_t *arena_dalloc_junk_large_orig; static huge_dalloc_junk_t *huge_dalloc_junk_orig; -static void *most_recently_junked; +static void *watch_for_junking; +static bool saw_junking; + +static void +watch_junking(void *p) +{ + + watch_for_junking = p; + saw_junking = false; +} static void arena_dalloc_junk_small_intercept(void *ptr, arena_bin_info_t *bin_info) @@ -21,7 +30,8 @@ arena_dalloc_junk_small_intercept(void *ptr, arena_bin_info_t *bin_info) "Missing junk fill for byte %zu/%zu of deallocated region", i, bin_info->reg_size); } - most_recently_junked = ptr; + if (ptr == watch_for_junking) + saw_junking = true; } static void @@ -35,7 +45,8 @@ arena_dalloc_junk_large_intercept(void *ptr, size_t usize) "Missing junk fill for byte %zu/%zu of deallocated region", i, usize); } - most_recently_junked = ptr; + if (ptr == watch_for_junking) + saw_junking = true; } static void @@ -48,7 +59,8 @@ huge_dalloc_junk_intercept(void *ptr, size_t usize) * enough that it doesn't make sense to duplicate the decision logic in * test code, so don't actually check that the region is junk-filled. */ - most_recently_junked = ptr; + if (ptr == watch_for_junking) + saw_junking = true; } static void @@ -87,18 +99,19 @@ test_junk(size_t sz_min, size_t sz_max) } if (xallocx(s, sz+1, 0, 0) == sz) { - void *junked = (void *)s; + watch_junking(s); s = (char *)rallocx(s, sz+1, 0); assert_ptr_not_null((void *)s, "Unexpected rallocx() failure"); - assert_ptr_eq(most_recently_junked, junked, + assert_true(saw_junking, "Expected region of size %zu to be junk-filled", sz); } } + watch_junking(s); dallocx(s, 0); - assert_ptr_eq(most_recently_junked, (void *)s, + assert_true(saw_junking, "Expected region of size %zu to be junk-filled", sz); arena_dalloc_junk_small = arena_dalloc_junk_small_orig; From 3a8b9b1fd95b1bb9b3dc00f6798eeb40d5100b7b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 8 Oct 2014 00:54:16 -0700 Subject: [PATCH 144/352] Fix a recursive lock acquisition regression. Fix a recursive lock acquisition regression, which was introduced by 8bb3198f72fc7587dc93527f9f19fb5be52fa553 (Refactor/fix arenas manipulation.). --- src/jemalloc.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 38b5aaf7..c62d8ce6 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -244,13 +244,11 @@ a0free(void *ptr) } /* Create a new arena and insert it into the arenas array at index ind. */ -arena_t * -arena_init(unsigned ind) +static arena_t * +arena_init_locked(unsigned ind) { arena_t *arena; - malloc_mutex_lock(&arenas_lock); - /* Expand arenas if necessary. */ assert(ind <= narenas_total); if (ind == narenas_total) { @@ -258,10 +256,8 @@ arena_init(unsigned ind) arena_t **arenas_new = (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * sizeof(arena_t *))); - if (arenas_new == NULL) { - arena = NULL; - goto label_return; - } + if (arenas_new == NULL) + return (NULL); memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); arenas_new[ind] = NULL; /* @@ -281,12 +277,21 @@ arena_init(unsigned ind) arena = arenas[ind]; if (arena != NULL) { assert(ind < narenas_auto); - goto label_return; + return (arena); } /* Actually initialize the arena. */ arena = arenas[ind] = arena_new(ind); -label_return: + return (arena); +} + +arena_t * +arena_init(unsigned ind) +{ + arena_t *arena; + + malloc_mutex_lock(&arenas_lock); + arena = arena_init_locked(ind); malloc_mutex_unlock(&arenas_lock); return (arena); } @@ -477,7 +482,7 @@ arena_choose_hard(tsd_t *tsd) } else { /* Initialize a new arena. */ choose = first_null; - ret = arena_init(choose); + ret = arena_init_locked(choose); if (ret == NULL) { malloc_mutex_unlock(&arenas_lock); return (NULL); From 57efa7bb0e284805c940472190bc9924327635a1 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 8 Oct 2014 17:57:19 -0700 Subject: [PATCH 145/352] Avoid atexit(3) when possible, disable prof_final by default. atexit(3) can deadlock internally during its own initialization if jemalloc calls atexit() during jemalloc initialization. Mitigate the impact by restructuring prof initialization to avoid calling atexit() unless the registered function will actually dump a final heap profile. Additionally, disable prof_final by default so that this land mine is opt-in rather than opt-out. This resolves #144. --- doc/jemalloc.xml.in | 18 +++++++++++++++--- src/prof.c | 17 +++++++++-------- test/unit/prof_active.c | 2 +- test/unit/prof_thread_name.c | 3 +-- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 1f692f78..7da1498a 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -857,8 +857,14 @@ for (i = 0; i < nbins; i++) { is specified during configuration, this has the potential to cause deadlock for a multi-threaded process that exits while one or more threads are executing in the memory allocation - functions. Therefore, this option should only be used with care; it is - primarily intended as a performance tuning aid during application + functions. Furthermore, atexit may + allocate memory during application initialization and then deadlock + internally when jemalloc in turn calls + atexit, so this option is not + univerally usable (though the application can register its own + atexit function with equivalent + functionality). Therefore, this option should only be used with care; + it is primarily intended as a performance tuning aid during application development. This option is disabled by default. @@ -1155,7 +1161,13 @@ malloc_conf = "xmalloc:true";]]> <prefix>.<pid>.<seq>.f.heap, where <prefix> is controlled by the opt.prof_prefix - option. This option is enabled by default. + option. Note that atexit may allocate + memory during application initialization and then deadlock internally + when jemalloc in turn calls atexit, so + this option is not univerally usable (though the application can + register its own atexit function with + equivalent functionality). This option is disabled by + default. diff --git a/src/prof.c b/src/prof.c index b3150a27..3e2e4277 100644 --- a/src/prof.c +++ b/src/prof.c @@ -20,7 +20,7 @@ bool opt_prof_thread_active_init = true; size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; bool opt_prof_gdump = false; -bool opt_prof_final = true; +bool opt_prof_final = false; bool opt_prof_leak = false; bool opt_prof_accum = false; char opt_prof_prefix[ @@ -1487,17 +1487,17 @@ prof_fdump(void) char filename[DUMP_FILENAME_BUFSIZE]; cassert(config_prof); + assert(opt_prof_final); + assert(opt_prof_prefix[0] != '\0'); if (!prof_booted) return; tsd = tsd_fetch(); - if (opt_prof_final && opt_prof_prefix[0] != '\0') { - malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename, 'f', VSEQ_INVALID); - malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(tsd, false, filename, opt_prof_leak); - } + malloc_mutex_lock(&prof_dump_seq_mtx); + prof_dump_filename(filename, 'f', VSEQ_INVALID); + malloc_mutex_unlock(&prof_dump_seq_mtx); + prof_dump(tsd, false, filename, opt_prof_leak); } void @@ -2023,7 +2023,8 @@ prof_boot2(void) if (malloc_mutex_init(&prof_dump_mtx)) return (true); - if (atexit(prof_fdump) != 0) { + if (opt_prof_final && opt_prof_prefix[0] != '\0' && + atexit(prof_fdump) != 0) { malloc_write(": Error in atexit()\n"); if (opt_abort) abort(); diff --git a/test/unit/prof_active.c b/test/unit/prof_active.c index d4bab8d0..81490957 100644 --- a/test/unit/prof_active.c +++ b/test/unit/prof_active.c @@ -2,7 +2,7 @@ #ifdef JEMALLOC_PROF const char *malloc_conf = - "prof:true,prof_thread_active_init:false,lg_prof_sample:0,prof_final:false"; + "prof:true,prof_thread_active_init:false,lg_prof_sample:0"; #endif static void diff --git a/test/unit/prof_thread_name.c b/test/unit/prof_thread_name.c index 6066dba7..f501158d 100644 --- a/test/unit/prof_thread_name.c +++ b/test/unit/prof_thread_name.c @@ -1,8 +1,7 @@ #include "test/jemalloc_test.h" #ifdef JEMALLOC_PROF -const char *malloc_conf = - "prof:true,prof_active:false,prof_final:false"; +const char *malloc_conf = "prof:true,prof_active:false"; #endif static void From b123ddc760e5b53dde17c6a19a130173067c0e30 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 8 Oct 2014 18:18:03 -0700 Subject: [PATCH 146/352] Don't configure HAVE_SSE2. Don't configure HAVE_SSE2 (on behalf of SFMT), because its dependencies are notoriously unportable in practice. This resolves #119. --- configure.ac | 10 ---------- test/include/test/jemalloc_test_defs.h.in | 5 ++++- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/configure.ac b/configure.ac index e4afe889..1d79ded6 100644 --- a/configure.ac +++ b/configure.ac @@ -206,8 +206,6 @@ AC_CANONICAL_HOST dnl CPU-specific settings. CPU_SPINWAIT="" case "${host_cpu}" in - i[[345]]86) - ;; i686|x86_64) JE_COMPILABLE([pause instruction], [], [[__asm__ volatile("pause"); return 0;]], @@ -215,14 +213,6 @@ case "${host_cpu}" in if test "x${je_cv_pause}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi - dnl emmintrin.h fails to compile unless MMX, SSE, and SSE2 are - dnl supported. - JE_COMPILABLE([SSE2 intrinsics], [ -#include -], [], [je_cv_sse2]) - if test "x${je_cv_sse2}" = "xyes" ; then - AC_DEFINE_UNQUOTED([HAVE_SSE2], [ ]) - fi ;; powerpc) AC_DEFINE_UNQUOTED([HAVE_ALTIVEC], [ ]) diff --git a/test/include/test/jemalloc_test_defs.h.in b/test/include/test/jemalloc_test_defs.h.in index aaaaec14..5cc8532a 100644 --- a/test/include/test/jemalloc_test_defs.h.in +++ b/test/include/test/jemalloc_test_defs.h.in @@ -1,6 +1,9 @@ #include "jemalloc/internal/jemalloc_internal_defs.h" #include "jemalloc/internal/jemalloc_internal_decls.h" -/* For use by SFMT. */ +/* + * For use by SFMT. configure.ac doesn't actually define HAVE_SSE2 because its + * dependencies are notoriously unportable in practice. + */ #undef HAVE_SSE2 #undef HAVE_ALTIVEC From fc0b3b7383373d66cfed2cd4e2faa272a6868d32 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 9 Oct 2014 17:54:06 -0700 Subject: [PATCH 147/352] Add configure options. Add: --with-lg-page --with-lg-page-sizes --with-lg-size-class-group --with-lg-quantum Get rid of STATIC_PAGE_SHIFT, in favor of directly setting LG_PAGE. Fix various edge conditions exposed by the configure options. --- INSTALL | 67 +++++++++++++++++++ Makefile.in | 1 + configure.ac | 47 ++++++++++--- include/jemalloc/internal/arena.h | 8 ++- include/jemalloc/internal/huge.h | 9 +-- .../jemalloc/internal/jemalloc_internal.h.in | 28 ++++---- .../internal/jemalloc_internal_defs.h.in | 10 ++- include/jemalloc/internal/private_symbols.txt | 4 +- include/jemalloc/internal/size_classes.sh | 12 +++- include/jemalloc/internal/tcache.h | 4 +- src/arena.c | 42 +++++++++--- src/huge.c | 52 +++++++------- src/jemalloc.c | 50 ++++++++------ src/tcache.c | 53 ++++----------- test/unit/lg_chunk.c | 26 +++++++ test/unit/mallctl.c | 2 +- 16 files changed, 278 insertions(+), 137 deletions(-) create mode 100644 test/unit/lg_chunk.c diff --git a/INSTALL b/INSTALL index 9af23369..73bf7185 100644 --- a/INSTALL +++ b/INSTALL @@ -189,6 +189,73 @@ any of the following arguments (not a definitive list) to 'configure': Specify where to find DocBook XSL stylesheets when building the documentation. +--with-lg-page= + Specify the base 2 log of the system page size. This option is only useful + when cross compiling, since the configure script automatically determines the + host's page size by default. + +--with-lg-page-sizes= + Specify the comma-separated base 2 logs of the page sizes to support. This + option may be useful when cross-compiling in combination with + --with-lg-page, but its primary use case is for integration with FreeBSD's + libc, wherein jemalloc is embedded. + +--with-lg-size-class-group= + Specify the base 2 log of how many size classes to use for each doubling in + size. By default jemalloc uses =2, which results in + e.g. the following size classes: + + [...], 64, + 80, 96, 112, 128, + 160, [...] + + =3 results in e.g. the following size classes: + + [...], 64, + 72, 80, 88, 96, 104, 112, 120, 128, + 144, [...] + + The minimal =0 causes jemalloc to only provide size + classes that are powers of 2: + + [...], + 64, + 128, + 256, + [...] + + An implementation detail currently limits the total number of small size + classes to 255, and a compilation error will result if the + you specify cannot be supported. The limit is + roughly =4, depending on page size. + +--with-lg-quantum= + Specify the base 2 log of the minimum allocation alignment (only + =3 and =4 are supported). jemalloc needs to know + the minimum alignment that meets the following C standard requirement + (quoted from the April 12, 2011 draft of the C11 standard): + + The pointer returned if the allocation succeeds is suitably aligned so + that it may be assigned to a pointer to any type of object with a + fundamental alignment requirement and then used to access such an object + or an array of such objects in the space allocated [...] + + This setting is architecture-specific, and although jemalloc includes known + safe values for the most commonly used modern architectures, there is a + wrinkle related to GNU libc (glibc) that may impact your choice of + . On most modern architectures, this mandates 16-byte alignment + (=4), but the glibc developers chose not to meet this requirement + for performance reasons. An old discussion can be found at + https://sourceware.org/bugzilla/show_bug.cgi?id=206 . Unlike glibc, + jemalloc does follow the C standard by default (caveat: jemalloc technically + cheats by only providing 8-byte alignment for 8-byte allocation requests), + but the fact that Linux systems already work around this allocator + noncompliance means that it is generally safe in practice to let jemalloc's + minimum alignment follow glibc's lead. If you specify --with-lg-quantum=3 + during configuration, jemalloc will provide additional size classes that + are not 16-byte-aligned (24, 40, and 56, assuming + --with-lg-size-class-group=2). + The following environment variables (not a definitive list) impact configure's behavior: diff --git a/Makefile.in b/Makefile.in index 50f6596a..40644ce8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -118,6 +118,7 @@ TESTS_UNIT := $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/hash.c \ $(srcroot)test/unit/junk.c \ + $(srcroot)test/unit/lg_chunk.c \ $(srcroot)test/unit/mallctl.c \ $(srcroot)test/unit/math.c \ $(srcroot)test/unit/mq.c \ diff --git a/configure.ac b/configure.ac index 1d79ded6..f8c09c46 100644 --- a/configure.ac +++ b/configure.ac @@ -969,8 +969,17 @@ else fi fi -AC_CACHE_CHECK([STATIC_PAGE_SHIFT], - [je_cv_static_page_shift], +AC_ARG_WITH([lg_quantum], + [AS_HELP_STRING([--with-lg-quantum=], + [Base 2 log of minimum allocation alignment])], + [AC_DEFINE_UNQUOTED([LG_QUANTUM], [$with_lg_quantum])]) + +AC_ARG_WITH([lg_page], + [AS_HELP_STRING([--with-lg-page=], [Base 2 log of system page size])], + [LG_PAGE="$with_lg_page"], [LG_PAGE="detect"]) +if test "x$LG_PAGE" == "xdetect"; then + AC_CACHE_CHECK([LG_PAGE], + [je_cv_lg_page], AC_RUN_IFELSE([AC_LANG_PROGRAM( [[ #include @@ -1006,15 +1015,29 @@ AC_CACHE_CHECK([STATIC_PAGE_SHIFT], return 0; ]])], - [je_cv_static_page_shift=`cat conftest.out`], - [je_cv_static_page_shift=undefined], - [je_cv_static_page_shift=12])) - -if test "x$je_cv_static_page_shift" != "xundefined"; then - AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$je_cv_static_page_shift]) -else - AC_MSG_ERROR([cannot determine value for STATIC_PAGE_SHIFT]) + [je_cv_lg_page=`cat conftest.out`], + [je_cv_lg_page=undefined], + [je_cv_lg_page=12])) fi +if test "x${je_cv_lg_page}" != "x" ; then + LG_PAGE="${je_cv_lg_page}" +fi +if test "x${LG_PAGE}" != "xundefined" ; then + AC_DEFINE_UNQUOTED([LG_PAGE], [$LG_PAGE]) +else + AC_MSG_ERROR([cannot determine value for LG_PAGE]) +fi + +AC_ARG_WITH([lg_page_sizes], + [AS_HELP_STRING([--with-lg-page-sizes=], + [Base 2 logs of system page sizes to support])], + [LG_PAGE_SIZES="$with_lg_page_sizes"], [LG_PAGE_SIZES="$LG_PAGE"]) + +AC_ARG_WITH([lg_size_class_group], + [AS_HELP_STRING([--with-lg-size-class-group=], + [Base 2 log of size classes per doubling])], + [LG_SIZE_CLASS_GROUP="$with_lg_size_class_group"], + [LG_SIZE_CLASS_GROUP="2"]) dnl ============================================================================ dnl jemalloc configuration. @@ -1456,10 +1479,12 @@ AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [ ]) AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ mkdir -p "${objroot}include/jemalloc/internal" - "${srcdir}/include/jemalloc/internal/size_classes.sh" > "${objroot}include/jemalloc/internal/size_classes.h" + "${srcdir}/include/jemalloc/internal/size_classes.sh" ${LG_PAGE_SIZES} ${LG_SIZE_CLASS_GROUP} > "${objroot}include/jemalloc/internal/size_classes.h" ], [ srcdir="${srcdir}" objroot="${objroot}" + LG_PAGE_SIZES=${LG_PAGE_SIZES} + LG_SIZE_CLASS_GROUP=${LG_SIZE_CLASS_GROUP} ]) AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [ mkdir -p "${objroot}include/jemalloc" diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 894ce9af..f5b9fc62 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -362,8 +362,8 @@ void *arena_malloc_small(arena_t *arena, size_t size, bool zero); void *arena_malloc_large(arena_t *arena, size_t size, bool zero); void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); void arena_prof_promoted(const void *ptr, size_t size); -void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, - arena_chunk_map_bits_t *bitselm); +void arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, + void *ptr, arena_chunk_map_bits_t *bitselm); void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm); void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, @@ -371,8 +371,10 @@ void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, #ifdef JEMALLOC_JET typedef void (arena_dalloc_junk_large_t)(void *, size_t); extern arena_dalloc_junk_large_t *arena_dalloc_junk_large; +#else +void arena_dalloc_junk_large(void *ptr, size_t usize); #endif -void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, +void arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr); void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); #ifdef JEMALLOC_JET diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 5d4d3a16..39d8aa50 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -9,19 +9,20 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero); +void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, + bool try_tcache); void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, - bool zero); + bool zero, bool try_tcache); bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, - bool try_tcache_dalloc); + bool try_tcache_alloc, bool try_tcache_dalloc); #ifdef JEMALLOC_JET typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; #endif -void huge_dalloc(tsd_t *tsd, void *ptr); +void huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache); size_t huge_salloc(const void *ptr); prof_tctx_t *huge_prof_tctx_get(const void *ptr); void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index f4d5de6a..3f65fad0 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -185,7 +185,7 @@ typedef unsigned index_t; #define TINY_MIN (1U << LG_TINY_MIN) /* - * Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size * classes). */ #ifndef LG_QUANTUM @@ -235,7 +235,8 @@ typedef unsigned index_t; # define LG_QUANTUM 4 # endif # ifndef LG_QUANTUM -# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS" +# error "Unknown minimum alignment for architecture; specify via " + "--with-lg-quantum" # endif #endif @@ -275,12 +276,11 @@ typedef unsigned index_t; #define CACHELINE_CEILING(s) \ (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) -/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */ +/* Page size. LG_PAGE is determined by the configure script. */ #ifdef PAGE_MASK # undef PAGE_MASK #endif -#define LG_PAGE STATIC_PAGE_SHIFT -#define PAGE ((size_t)(1U << STATIC_PAGE_SHIFT)) +#define PAGE ((size_t)(1U << LG_PAGE)) #define PAGE_MASK ((size_t)(PAGE - 1)) /* Return the smallest pagesize multiple that is >= s. */ @@ -809,7 +809,7 @@ imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) if (size <= arena_maxclass) return (arena_malloc(tsd, arena, size, false, try_tcache)); else - return (huge_malloc(tsd, arena, size, false)); + return (huge_malloc(tsd, arena, size, false, try_tcache)); } JEMALLOC_ALWAYS_INLINE void * @@ -826,7 +826,7 @@ icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) if (size <= arena_maxclass) return (arena_malloc(tsd, arena, size, true, try_tcache)); else - return (huge_malloc(tsd, arena, size, true)); + return (huge_malloc(tsd, arena, size, true, try_tcache)); } JEMALLOC_ALWAYS_INLINE void * @@ -854,9 +854,11 @@ ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, return (NULL); ret = arena_palloc(arena, usize, alignment, zero); } else if (alignment <= chunksize) - ret = huge_malloc(tsd, arena, usize, zero); - else - ret = huge_palloc(tsd, arena, usize, alignment, zero); + ret = huge_malloc(tsd, arena, usize, zero, try_tcache); + else { + ret = huge_palloc(tsd, arena, usize, alignment, zero, + try_tcache); + } } assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); @@ -938,7 +940,7 @@ idalloct(tsd_t *tsd, void *ptr, bool try_tcache) if (chunk != ptr) arena_dalloc(tsd, chunk, ptr, try_tcache); else - huge_dalloc(tsd, ptr); + huge_dalloc(tsd, ptr, try_tcache); } JEMALLOC_ALWAYS_INLINE void @@ -952,7 +954,7 @@ isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) if (chunk != ptr) arena_sdalloc(tsd, chunk, ptr, size, try_tcache); else - huge_dalloc(tsd, ptr); + huge_dalloc(tsd, ptr, try_tcache); } JEMALLOC_ALWAYS_INLINE void @@ -1042,7 +1044,7 @@ iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero, alignment, zero, try_tcache_alloc, try_tcache_dalloc)); } else { return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0, - alignment, zero, try_tcache_dalloc)); + alignment, zero, try_tcache_alloc, try_tcache_dalloc)); } } diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index fd85e5cf..0ff939c6 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -144,8 +144,14 @@ /* Support lazy locking (avoid locking unless a second thread is launched). */ #undef JEMALLOC_LAZY_LOCK -/* One page is 2^STATIC_PAGE_SHIFT bytes. */ -#undef STATIC_PAGE_SHIFT +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#undef LG_QUANTUM + +/* One page is 2^LG_PAGE bytes. */ +#undef LG_PAGE /* * If defined, use munmap() to unmap freed chunks, rather than storing them for diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index d5e6fdcf..66d48221 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -16,11 +16,11 @@ arena_chunk_dalloc_huge arena_cleanup arena_dalloc arena_dalloc_bin -arena_dalloc_bin_locked +arena_dalloc_bin_junked_locked arena_dalloc_junk_large arena_dalloc_junk_small arena_dalloc_large -arena_dalloc_large_locked +arena_dalloc_large_junked_locked arena_dalloc_small arena_dss_prec_get arena_dss_prec_set diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 897570cc..733338c5 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -1,4 +1,6 @@ #!/bin/sh +# +# Usage: size_classes.sh # The following limits are chosen such that they cover all supported platforms. @@ -15,10 +17,10 @@ lg_tmin=3 lg_kmax=12 # Page sizes. -lg_parr="12 13 16" +lg_parr=`echo $1 | tr ',' ' '` # Size class group size (number of size classes for each size doubling). -lg_g=2 +lg_g=$2 pow2() { e=$1 @@ -159,7 +161,11 @@ size_classes() { nbins=$((${index} + 1)) # Final written value is correct: small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - lg_large_minclass=$((${lg_grp} + 1)) + if [ ${lg_g} -gt 0 ] ; then + lg_large_minclass=$((${lg_grp} + 1)) + else + lg_large_minclass=$((${lg_grp} + 2)) + fi fi index=$((${index} + 1)) ndelta=$((${ndelta} + 1)) diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 02eec5db..fe9c47e8 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -112,7 +112,7 @@ void tcache_arena_associate(tcache_t *tcache, arena_t *arena); void tcache_arena_reassociate(tcache_t *tcache, arena_t *arena); void tcache_arena_dissociate(tcache_t *tcache); tcache_t *tcache_get_hard(tsd_t *tsd); -tcache_t *tcache_create(arena_t *arena); +tcache_t *tcache_create(tsd_t *tsd, arena_t *arena); void tcache_cleanup(tsd_t *tsd); void tcache_enabled_cleanup(tsd_t *tsd); void tcache_stats_merge(tcache_t *tcache, arena_t *arena); @@ -363,7 +363,7 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) binind = size2index(size); if (config_fill && unlikely(opt_junk)) - memset(ptr, 0x5a, size); + arena_dalloc_junk_large(ptr, size); tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; diff --git a/src/arena.c b/src/arena.c index 86e54404..bbe58fa6 100644 --- a/src/arena.c +++ b/src/arena.c @@ -623,7 +623,7 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero) arena_chunk_t *chunk; arena_run_t *run; - assert(size <= arena_maxclass); + assert(size <= arena_maxrun); assert((size & PAGE_MASK) == 0); /* Search the arena's chunks for the lowest best fit. */ @@ -673,7 +673,7 @@ arena_run_alloc_small(arena_t *arena, size_t size, index_t binind) arena_chunk_t *chunk; arena_run_t *run; - assert(size <= arena_maxclass); + assert(size <= arena_maxrun); assert((size & PAGE_MASK) == 0); assert(binind != BININD_INVALID); @@ -1728,9 +1728,9 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_runs_insert(bin, run); } -void -arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, - arena_chunk_map_bits_t *bitselm) +static void +arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, + arena_chunk_map_bits_t *bitselm, bool junked) { size_t pageind, rpages_ind; arena_run_t *run; @@ -1749,7 +1749,7 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, if (config_fill || config_stats) size = bin_info->reg_size; - if (config_fill && unlikely(opt_junk)) + if (!junked && config_fill && unlikely(opt_junk)) arena_dalloc_junk_small(ptr, bin_info); arena_run_reg_dalloc(run, ptr); @@ -1765,6 +1765,14 @@ arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, } } +void +arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, + arena_chunk_map_bits_t *bitselm) +{ + + arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, true); +} + void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind, arena_chunk_map_bits_t *bitselm) @@ -1777,7 +1785,7 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, run = &arena_miscelm_get(chunk, rpages_ind)->run; bin = run->bin; malloc_mutex_lock(&bin->lock); - arena_dalloc_bin_locked(arena, chunk, ptr, bitselm); + arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false); malloc_mutex_unlock(&bin->lock); } @@ -1800,7 +1808,7 @@ arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, #undef arena_dalloc_junk_large #define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) #endif -static void +void arena_dalloc_junk_large(void *ptr, size_t usize) { @@ -1815,7 +1823,8 @@ arena_dalloc_junk_large_t *arena_dalloc_junk_large = #endif void -arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) +arena_dalloc_large_locked_impl(arena_t *arena, arena_chunk_t *chunk, + void *ptr, bool junked) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); @@ -1824,7 +1833,8 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) if (config_fill || config_stats) { size_t usize = arena_mapbits_large_size_get(chunk, pageind); - arena_dalloc_junk_large(ptr, usize); + if (!junked) + arena_dalloc_junk_large(ptr, usize); if (config_stats) { index_t index = size2index(usize) - NBINS; @@ -1838,12 +1848,20 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) arena_run_dalloc(arena, run, true, false); } +void +arena_dalloc_large_junked_locked(arena_t *arena, arena_chunk_t *chunk, + void *ptr) +{ + + arena_dalloc_large_locked_impl(arena, chunk, ptr, true); +} + void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) { malloc_mutex_lock(&arena->lock); - arena_dalloc_large_locked(arena, chunk, ptr); + arena_dalloc_large_locked_impl(arena, chunk, ptr, false); malloc_mutex_unlock(&arena->lock); } @@ -2398,6 +2416,7 @@ arena_boot(void) sizeof(arena_chunk_map_bits_t) * (chunk_npages-map_bias); arena_maxrun = chunksize - (map_bias << LG_PAGE); + assert(arena_maxrun > 0); arena_maxclass = index2size(size2index(chunksize)-1); if (arena_maxclass > arena_maxrun) { /* @@ -2407,6 +2426,7 @@ arena_boot(void) */ arena_maxclass = arena_maxrun; } + assert(arena_maxclass > 0); nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS); bin_info_init(); diff --git a/src/huge.c b/src/huge.c index 541df60a..6c9b97bb 100644 --- a/src/huge.c +++ b/src/huge.c @@ -13,7 +13,7 @@ static malloc_mutex_t huge_mtx; static extent_tree_t huge; void * -huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero) +huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, bool try_tcache) { size_t usize; @@ -23,12 +23,12 @@ huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero) return (NULL); } - return (huge_palloc(tsd, arena, usize, chunksize, zero)); + return (huge_palloc(tsd, arena, usize, chunksize, zero, try_tcache)); } void * huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, - bool zero) + bool zero, bool try_tcache) { void *ret; size_t csize; @@ -42,7 +42,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, /* Allocate an extent node with which to track the chunk. */ node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), - CACHELINE, false, tsd != NULL, NULL); + CACHELINE, false, try_tcache, NULL); if (node == NULL) return (NULL); @@ -58,7 +58,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, } ret = arena_chunk_alloc_huge(arena, NULL, csize, alignment, &is_zeroed); if (ret == NULL) { - idalloct(tsd, node, tsd != NULL); + idalloct(tsd, node, try_tcache); return (NULL); } @@ -122,6 +122,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { expand_addr = ptr + CHUNK_CEILING(oldsize); expand_size = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); + assert(expand_size > 0); malloc_mutex_lock(&huge_mtx); @@ -223,13 +224,8 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, return (false); } - if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) - && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { - return (false); - } - /* Shrink the allocation in-place. */ - if (CHUNK_CEILING(oldsize) > CHUNK_CEILING(usize)) { + if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)) { extent_node_t *node, key; void *excess_addr; size_t excess_size; @@ -251,7 +247,10 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, /* Zap the excess chunks. */ huge_dalloc_junk(ptr + usize, oldsize - usize); - arena_chunk_dalloc_huge(node->arena, excess_addr, excess_size); + if (excess_size > 0) { + arena_chunk_dalloc_huge(node->arena, excess_addr, + excess_size); + } return (false); } @@ -269,7 +268,8 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, void * huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_dalloc) + size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc) { void *ret; size_t copysize; @@ -283,19 +283,25 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, * different size class. In that case, fall back to allocating new * space and copying. */ - if (alignment > chunksize) - ret = huge_palloc(tsd, arena, size + extra, alignment, zero); - else - ret = huge_malloc(tsd, arena, size + extra, zero); + if (alignment > chunksize) { + ret = huge_palloc(tsd, arena, size + extra, alignment, zero, + try_tcache_alloc); + } else { + ret = huge_malloc(tsd, arena, size + extra, zero, + try_tcache_alloc); + } if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ - if (alignment > chunksize) - ret = huge_palloc(tsd, arena, size, alignment, zero); - else - ret = huge_malloc(tsd, arena, size, zero); + if (alignment > chunksize) { + ret = huge_palloc(tsd, arena, size, alignment, zero, + try_tcache_alloc); + } else { + ret = huge_malloc(tsd, arena, size, zero, + try_tcache_alloc); + } if (ret == NULL) return (NULL); @@ -312,7 +318,7 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, } void -huge_dalloc(tsd_t *tsd, void *ptr) +huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache) { extent_node_t *node, key; @@ -330,7 +336,7 @@ huge_dalloc(tsd_t *tsd, void *ptr) huge_dalloc_junk(node->addr, node->size); arena_chunk_dalloc_huge(node->arena, node->addr, CHUNK_CEILING(node->size)); - idalloct(tsd, node, tsd != NULL); + idalloct(tsd, node, try_tcache); } size_t diff --git a/src/jemalloc.c b/src/jemalloc.c index c62d8ce6..a862104a 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -67,6 +67,8 @@ const uint8_t size2index_tab[] = { #define S2B_7(i) S2B_6(i) S2B_6(i) #define S2B_8(i) S2B_7(i) S2B_7(i) #define S2B_9(i) S2B_8(i) S2B_8(i) +#define S2B_10(i) S2B_9(i) S2B_9(i) +#define S2B_11(i) S2B_10(i) S2B_10(i) #define S2B_no(i) #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ S2B_##lg_delta_lookup(index) @@ -78,6 +80,8 @@ const uint8_t size2index_tab[] = { #undef S2B_7 #undef S2B_8 #undef S2B_9 +#undef S2B_10 +#undef S2B_11 #undef S2B_no #undef SC }; @@ -199,6 +203,7 @@ static void * a0alloc(size_t size, bool zero) { void *ret; + tsd_t *tsd; if (unlikely(malloc_init())) return (NULL); @@ -206,10 +211,11 @@ a0alloc(size_t size, bool zero) if (size == 0) size = 1; + tsd = tsd_fetch(); if (size <= arena_maxclass) - ret = arena_malloc(NULL, a0get(), size, zero, false); + ret = arena_malloc(tsd, a0get(), size, zero, false); else - ret = huge_malloc(NULL, a0get(), size, zero); + ret = huge_malloc(tsd, a0get(), size, zero, false); return (ret); } @@ -231,16 +237,18 @@ a0calloc(size_t num, size_t size) void a0free(void *ptr) { + tsd_t *tsd; arena_chunk_t *chunk; if (ptr == NULL) return; + tsd = tsd_fetch(); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(NULL, chunk, ptr, false); + arena_dalloc(tsd, chunk, ptr, false); else - huge_dalloc(NULL, ptr); + huge_dalloc(tsd, ptr, false); } /* Create a new arena and insert it into the arenas array at index ind. */ @@ -817,15 +825,15 @@ malloc_conf_init(void) "Invalid conf value", \ k, klen, v, vlen); \ } else if (clip) { \ - if (min != 0 && um < min) \ - o = min; \ - else if (um > max) \ - o = max; \ + if ((min) != 0 && um < (min)) \ + o = (min); \ + else if (um > (max)) \ + o = (max); \ else \ o = um; \ } else { \ - if ((min != 0 && um < min) || \ - um > max) { \ + if (((min) != 0 && um < (min)) \ + || um > (max)) { \ malloc_conf_error( \ "Out-of-range " \ "conf value", \ @@ -847,8 +855,8 @@ malloc_conf_init(void) malloc_conf_error( \ "Invalid conf value", \ k, klen, v, vlen); \ - } else if (l < (ssize_t)min || l > \ - (ssize_t)max) { \ + } else if (l < (ssize_t)(min) || l > \ + (ssize_t)(max)) { \ malloc_conf_error( \ "Out-of-range conf value", \ k, klen, v, vlen); \ @@ -868,15 +876,16 @@ malloc_conf_init(void) CONF_HANDLE_BOOL(opt_abort, "abort", true) /* - * Chunks always require at least one header page, plus - * one data page in the absence of redzones, or three - * pages in the presence of redzones. In order to - * simplify options processing, fix the limit based on - * config_fill. + * Chunks always require at least one header page, + * as many as 2^(LG_SIZE_CLASS_GROUP+1) data pages, and + * possibly an additional page in the presence of + * redzones. In order to simplify options processing, + * use a conservative bound that accommodates all these + * constraints. */ CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + - (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1, - true) + LG_SIZE_CLASS_GROUP + (config_fill ? 2 : 1), + (sizeof(size_t) << 3) - 1, true) if (strncmp("dss", k, klen) == 0) { int i; bool match = false; @@ -2088,8 +2097,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); - // XX Dangerous arenas read. - arena = arenas[arena_ind]; + arena = arena_get(tsd, arena_ind, true, true); } else arena = NULL; diff --git a/src/tcache.c b/src/tcache.c index 1bf70269..34224ec4 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -117,8 +117,8 @@ tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_bits_t *bitselm = arena_bitselm_get(chunk, pageind); - arena_dalloc_bin_locked(arena, chunk, ptr, - bitselm); + arena_dalloc_bin_junked_locked(arena, chunk, + ptr, bitselm); } else { /* * This object was allocated via a different @@ -193,9 +193,10 @@ tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->arena == arena) - arena_dalloc_large_locked(arena, chunk, ptr); - else { + if (chunk->arena == arena) { + arena_dalloc_large_junked_locked(arena, chunk, + ptr); + } else { /* * This object was allocated via a different * arena than the one that is currently locked. @@ -279,11 +280,11 @@ tcache_get_hard(tsd_t *tsd) arena = arena_choose(tsd, NULL); if (unlikely(arena == NULL)) return (NULL); - return (tcache_create(arena)); + return (tcache_create(tsd, arena)); } tcache_t * -tcache_create(arena_t *arena) +tcache_create(tsd_t *tsd, arena_t *arena) { tcache_t *tcache; size_t size, stack_offset; @@ -294,23 +295,10 @@ tcache_create(arena_t *arena) size = PTR_CEILING(size); stack_offset = size; size += stack_nelms * sizeof(void *); - /* - * Round up to the nearest multiple of the cacheline size, in order to - * avoid the possibility of false cacheline sharing. - * - * That this works relies on the same logic as in ipalloc(), but we - * cannot directly call ipalloc() here due to tcache bootstrapping - * issues. - */ - size = (size + CACHELINE_MASK) & (-CACHELINE); - - if (size <= SMALL_MAXCLASS) - tcache = (tcache_t *)arena_malloc_small(arena, size, true); - else if (size <= tcache_maxclass) - tcache = (tcache_t *)arena_malloc_large(arena, size, true); - else - tcache = (tcache_t *)icalloct(NULL, size, false, arena); + /* Avoid false cacheline sharing. */ + size = sa2u(size, CACHELINE); + tcache = ipalloct(tsd, size, CACHELINE, true, false, arena); if (tcache == NULL) return (NULL); @@ -331,7 +319,6 @@ static void tcache_destroy(tsd_t *tsd, tcache_t *tcache) { unsigned i; - size_t tcache_size; tcache_arena_dissociate(tcache); @@ -366,23 +353,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) arena_prof_accum(tcache->arena, tcache->prof_accumbytes)) prof_idump(); - tcache_size = arena_salloc(tcache, false); - if (tcache_size <= SMALL_MAXCLASS) { - arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); - arena_t *arena = chunk->arena; - size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> - LG_PAGE; - arena_chunk_map_bits_t *bitselm = arena_bitselm_get(chunk, - pageind); - - arena_dalloc_bin(arena, chunk, tcache, pageind, bitselm); - } else if (tcache_size <= tcache_maxclass) { - arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); - arena_t *arena = chunk->arena; - - arena_dalloc_large(arena, chunk, tcache); - } else - idalloct(tsd, tcache, false); + idalloct(tsd, tcache, false); } void diff --git a/test/unit/lg_chunk.c b/test/unit/lg_chunk.c new file mode 100644 index 00000000..7f0b31ce --- /dev/null +++ b/test/unit/lg_chunk.c @@ -0,0 +1,26 @@ +#include "test/jemalloc_test.h" + +/* + * Make sure that opt.lg_chunk clamping is sufficient. In practice, this test + * program will fail a debug assertion during initialization and abort (rather + * than the test soft-failing) if clamping is insufficient. + */ +const char *malloc_conf = "lg_chunk:0"; + +TEST_BEGIN(test_lg_chunk_clamp) +{ + void *p; + + p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + dallocx(p, 0); +} +TEST_END + +int +main(void) +{ + + return (test( + test_lg_chunk_clamp)); +} diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index e62e54f2..a8f7aed6 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -357,7 +357,7 @@ TEST_BEGIN(test_arenas_lrun_constants) assert_zu_eq(name, expected, "Incorrect "#name" size"); \ } while (0) - TEST_ARENAS_LRUN_CONSTANT(size_t, size, (1 << (LG_PAGE+2))); + TEST_ARENAS_LRUN_CONSTANT(size_t, size, LARGE_MINCLASS); #undef TEST_ARENAS_LRUN_CONSTANT } From 9b75677e538836b284a0d26a593963187c24a153 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 10 Oct 2014 18:19:20 -0700 Subject: [PATCH 148/352] Don't fetch tsd in a0{d,}alloc(). Don't fetch tsd in a0{d,}alloc(), because doing so can cause infinite recursion on systems that require an allocated tsd wrapper. --- src/jemalloc.c | 18 +++++++----------- test/unit/mq.c | 1 + 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index a862104a..fc490eba 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -203,7 +203,6 @@ static void * a0alloc(size_t size, bool zero) { void *ret; - tsd_t *tsd; if (unlikely(malloc_init())) return (NULL); @@ -211,11 +210,10 @@ a0alloc(size_t size, bool zero) if (size == 0) size = 1; - tsd = tsd_fetch(); if (size <= arena_maxclass) - ret = arena_malloc(tsd, a0get(), size, zero, false); + ret = arena_malloc(NULL, a0get(), size, zero, false); else - ret = huge_malloc(tsd, a0get(), size, zero, false); + ret = huge_malloc(NULL, a0get(), size, zero, false); return (ret); } @@ -237,18 +235,16 @@ a0calloc(size_t num, size_t size) void a0free(void *ptr) { - tsd_t *tsd; arena_chunk_t *chunk; if (ptr == NULL) return; - tsd = tsd_fetch(); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(tsd, chunk, ptr, false); + arena_dalloc(NULL, chunk, ptr, false); else - huge_dalloc(tsd, ptr, false); + huge_dalloc(NULL, ptr, false); } /* Create a new arena and insert it into the arenas array at index ind. */ @@ -2301,9 +2297,9 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) * fork/malloc races via the following functions it registers during * initialization using pthread_atfork(), but of course that does no good if * the allocator isn't fully initialized at fork time. The following library - * constructor is a partial solution to this problem. It may still possible to - * trigger the deadlock described above, but doing so would involve forking via - * a library constructor that runs before jemalloc's runs. + * constructor is a partial solution to this problem. It may still be possible + * to trigger the deadlock described above, but doing so would involve forking + * via a library constructor that runs before jemalloc's runs. */ JEMALLOC_ATTR(constructor) static void diff --git a/test/unit/mq.c b/test/unit/mq.c index bd289c54..bde2a480 100644 --- a/test/unit/mq.c +++ b/test/unit/mq.c @@ -85,6 +85,7 @@ TEST_END int main(void) { + return (test( test_mq_basic, test_mq_threaded)); From 2eb941a3d3a69fa8a73902b29564294f854fc3b0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 10 Oct 2014 20:40:43 -0700 Subject: [PATCH 149/352] Add AC_CACHE_CHECK() for pause instruction. This supports cross compilation. --- configure.ac | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/configure.ac b/configure.ac index f8c09c46..cc30da93 100644 --- a/configure.ac +++ b/configure.ac @@ -207,9 +207,10 @@ dnl CPU-specific settings. CPU_SPINWAIT="" case "${host_cpu}" in i686|x86_64) - JE_COMPILABLE([pause instruction], [], - [[__asm__ volatile("pause"); return 0;]], - [je_cv_pause]) + AC_CACHE_CHECK([whether pause instruction is compilable], [je_cv_pause], + [JE_COMPILABLE([pause instruction], [], + [[__asm__ volatile("pause"); return 0;]], + [je_cv_pause])]) if test "x${je_cv_pause}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi From 81e547566e9bd55db7c317c5848ab9dc189047cb Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 10 Oct 2014 22:34:25 -0700 Subject: [PATCH 150/352] Add --with-lg-tiny-min, generalize --with-lg-quantum. --- INSTALL | 32 ++++++++--- configure.ac | 21 ++++++-- .../jemalloc/internal/jemalloc_internal.h.in | 1 - .../internal/jemalloc_internal_defs.h.in | 3 ++ include/jemalloc/internal/size_classes.sh | 10 ++-- src/jemalloc.c | 54 +++++++++++++++++++ 6 files changed, 105 insertions(+), 16 deletions(-) diff --git a/INSTALL b/INSTALL index 73bf7185..a00960aa 100644 --- a/INSTALL +++ b/INSTALL @@ -230,10 +230,9 @@ any of the following arguments (not a definitive list) to 'configure': roughly =4, depending on page size. --with-lg-quantum= - Specify the base 2 log of the minimum allocation alignment (only - =3 and =4 are supported). jemalloc needs to know - the minimum alignment that meets the following C standard requirement - (quoted from the April 12, 2011 draft of the C11 standard): + Specify the base 2 log of the minimum allocation alignment. jemalloc needs + to know the minimum alignment that meets the following C standard + requirement (quoted from the April 12, 2011 draft of the C11 standard): The pointer returned if the allocation succeeds is suitably aligned so that it may be assigned to a pointer to any type of object with a @@ -247,8 +246,8 @@ any of the following arguments (not a definitive list) to 'configure': (=4), but the glibc developers chose not to meet this requirement for performance reasons. An old discussion can be found at https://sourceware.org/bugzilla/show_bug.cgi?id=206 . Unlike glibc, - jemalloc does follow the C standard by default (caveat: jemalloc technically - cheats by only providing 8-byte alignment for 8-byte allocation requests), + jemalloc does follow the C standard by default (caveat: jemalloc + technically cheats if --with-lg-tiny-min is smaller than --with-lg-quantum), but the fact that Linux systems already work around this allocator noncompliance means that it is generally safe in practice to let jemalloc's minimum alignment follow glibc's lead. If you specify --with-lg-quantum=3 @@ -256,6 +255,27 @@ any of the following arguments (not a definitive list) to 'configure': are not 16-byte-aligned (24, 40, and 56, assuming --with-lg-size-class-group=2). +--with-lg-tiny-min= + Specify the base 2 log of the minimum tiny size class to support. Tiny + size classes are powers of 2 less than the quantum, and are only + incorporated if is less than (see + --with-lg-quantum). Tiny size classes technically violate the C standard + requirement for minimum alignment, and crashes could conceivably result if + the compiler were to generate instructions that made alignment assumptions, + both because illegal instruction traps could result, and because accesses + could straddle page boundaries and cause segmentation faults due to + accessing unmapped addresses. + + The default of =3 works well in practice even on architectures + that technically require 16-byte alignment, probably for the same reason + --with-lg-quantum=3 works. Smaller tiny size classes can, and will, cause + crashes (see https://bugzilla.mozilla.org/show_bug.cgi?id=691003 for an + example). + + This option is rarely useful, and is mainly provided as documentation of a + subtle implementation detail. If you do use this option, specify a + value in [3, ..., ]. + The following environment variables (not a definitive list) impact configure's behavior: diff --git a/configure.ac b/configure.ac index cc30da93..a7bf1039 100644 --- a/configure.ac +++ b/configure.ac @@ -207,7 +207,7 @@ dnl CPU-specific settings. CPU_SPINWAIT="" case "${host_cpu}" in i686|x86_64) - AC_CACHE_CHECK([whether pause instruction is compilable], [je_cv_pause], + AC_CACHE_VAL([je_cv_pause], [JE_COMPILABLE([pause instruction], [], [[__asm__ volatile("pause"); return 0;]], [je_cv_pause])]) @@ -970,10 +970,21 @@ else fi fi +AC_ARG_WITH([lg_tiny_min], + [AS_HELP_STRING([--with-lg-tiny-min=], + [Base 2 log of minimum tiny size class to support])], + [LG_TINY_MIN="$with_lg_tiny_min"], + [LG_TINY_MIN="3"]) +AC_DEFINE_UNQUOTED([LG_TINY_MIN], [$LG_TINY_MIN]) + AC_ARG_WITH([lg_quantum], [AS_HELP_STRING([--with-lg-quantum=], [Base 2 log of minimum allocation alignment])], - [AC_DEFINE_UNQUOTED([LG_QUANTUM], [$with_lg_quantum])]) + [LG_QUANTA="$with_lg_quantum"], + [LG_QUANTA="3 4"]) +if test "x$with_lg_quantum" != "x" ; then + AC_DEFINE_UNQUOTED([LG_QUANTUM], [$with_lg_quantum]) +fi AC_ARG_WITH([lg_page], [AS_HELP_STRING([--with-lg-page=], [Base 2 log of system page size])], @@ -1480,11 +1491,13 @@ AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [ ]) AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ mkdir -p "${objroot}include/jemalloc/internal" - "${srcdir}/include/jemalloc/internal/size_classes.sh" ${LG_PAGE_SIZES} ${LG_SIZE_CLASS_GROUP} > "${objroot}include/jemalloc/internal/size_classes.h" + "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" ${LG_TINY_MIN} "${LG_PAGE_SIZES}" ${LG_SIZE_CLASS_GROUP} > "${objroot}include/jemalloc/internal/size_classes.h" ], [ srcdir="${srcdir}" objroot="${objroot}" - LG_PAGE_SIZES=${LG_PAGE_SIZES} + LG_QUANTA="${LG_QUANTA}" + LG_TINY_MIN=${LG_TINY_MIN} + LG_PAGE_SIZES="${LG_PAGE_SIZES}" LG_SIZE_CLASS_GROUP=${LG_SIZE_CLASS_GROUP} ]) AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 3f65fad0..294e2cc1 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -181,7 +181,6 @@ typedef unsigned index_t; (((unsigned)(flags >> 8)) - 1) /* Smallest size class to support. */ -#define LG_TINY_MIN 3 #define TINY_MIN (1U << LG_TINY_MIN) /* diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 0ff939c6..dccbb1ed 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -144,6 +144,9 @@ /* Support lazy locking (avoid locking unless a second thread is launched). */ #undef JEMALLOC_LAZY_LOCK +/* Minimum size class to support is 2^LG_TINY_MIN bytes. */ +#undef LG_TINY_MIN + /* * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size * classes). diff --git a/include/jemalloc/internal/size_classes.sh b/include/jemalloc/internal/size_classes.sh index 733338c5..38020dc6 100755 --- a/include/jemalloc/internal/size_classes.sh +++ b/include/jemalloc/internal/size_classes.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Usage: size_classes.sh +# Usage: size_classes.sh # The following limits are chosen such that they cover all supported platforms. @@ -8,19 +8,19 @@ lg_zarr="2 3" # Quanta. -lg_qarr="3 4" +lg_qarr=$1 # The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)]. -lg_tmin=3 +lg_tmin=$2 # Maximum lookup size. lg_kmax=12 # Page sizes. -lg_parr=`echo $1 | tr ',' ' '` +lg_parr=`echo $3 | tr ',' ' '` # Size class group size (number of size classes for each size doubling). -lg_g=$2 +lg_g=$4 pow2() { e=$1 diff --git a/src/jemalloc.c b/src/jemalloc.c index fc490eba..45439595 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -60,15 +60,69 @@ const size_t index2size_tab[NSIZES] = { JEMALLOC_ALIGNED(CACHELINE) const uint8_t size2index_tab[] = { +#if LG_TINY_MIN == 0 +#warning "Dangerous LG_TINY_MIN" +#define S2B_0(i) i, +#elif LG_TINY_MIN == 1 +#warning "Dangerous LG_TINY_MIN" +#define S2B_1(i) i, +#elif LG_TINY_MIN == 2 +#warning "Dangerous LG_TINY_MIN" +#define S2B_2(i) i, +#elif LG_TINY_MIN == 3 #define S2B_3(i) i, +#elif LG_TINY_MIN == 4 +#define S2B_4(i) i, +#elif LG_TINY_MIN == 5 +#define S2B_5(i) i, +#elif LG_TINY_MIN == 6 +#define S2B_6(i) i, +#elif LG_TINY_MIN == 7 +#define S2B_7(i) i, +#elif LG_TINY_MIN == 8 +#define S2B_8(i) i, +#elif LG_TINY_MIN == 9 +#define S2B_9(i) i, +#elif LG_TINY_MIN == 10 +#define S2B_10(i) i, +#elif LG_TINY_MIN == 11 +#define S2B_11(i) i, +#else +#error "Unsupported LG_TINY_MIN" +#endif +#if LG_TINY_MIN < 1 +#define S2B_1(i) S2B_0(i) S2B_0(i) +#endif +#if LG_TINY_MIN < 2 +#define S2B_2(i) S2B_1(i) S2B_1(i) +#endif +#if LG_TINY_MIN < 3 +#define S2B_3(i) S2B_2(i) S2B_2(i) +#endif +#if LG_TINY_MIN < 4 #define S2B_4(i) S2B_3(i) S2B_3(i) +#endif +#if LG_TINY_MIN < 5 #define S2B_5(i) S2B_4(i) S2B_4(i) +#endif +#if LG_TINY_MIN < 6 #define S2B_6(i) S2B_5(i) S2B_5(i) +#endif +#if LG_TINY_MIN < 7 #define S2B_7(i) S2B_6(i) S2B_6(i) +#endif +#if LG_TINY_MIN < 8 #define S2B_8(i) S2B_7(i) S2B_7(i) +#endif +#if LG_TINY_MIN < 9 #define S2B_9(i) S2B_8(i) S2B_8(i) +#endif +#if LG_TINY_MIN < 10 #define S2B_10(i) S2B_9(i) S2B_9(i) +#endif +#if LG_TINY_MIN < 11 #define S2B_11(i) S2B_10(i) S2B_10(i) +#endif #define S2B_no(i) #define SC(index, lg_grp, lg_delta, ndelta, bin, lg_delta_lookup) \ S2B_##lg_delta_lookup(index) From 381c23dd9d3bf019cc4c7523a900be1e888802a7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 10 Oct 2014 23:01:03 -0700 Subject: [PATCH 151/352] Remove arena_dalloc_bin_run() clean page preservation. Remove code in arena_dalloc_bin_run() that preserved the "clean" state of trailing clean pages by splitting them into a separate run during deallocation. This was a useful mechanism for reducing dirty page churn when bin runs comprised many pages, but bin runs are now quite small. Remove the nextind field from arena_run_t now that it is no longer needed, and change arena_run_t's bin field (arena_bin_t *) to binind (index_t). These two changes remove 8 bytes of chunk header overhead per page, which saves 1/512 of all arena chunk memory. --- include/jemalloc/internal/arena.h | 14 +++--- src/arena.c | 73 +++---------------------------- 2 files changed, 13 insertions(+), 74 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index f5b9fc62..28ff7271 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -36,11 +36,8 @@ typedef struct arena_s arena_t; #ifdef JEMALLOC_H_STRUCTS struct arena_run_s { - /* Bin this run is associated with. */ - arena_bin_t *bin; - - /* Index of next region that has never been allocated, or nregs. */ - uint32_t nextind; + /* Index of bin this run is associated with. */ + index_t binind; /* Number of free regions in run. */ unsigned nfree; @@ -756,7 +753,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) size_t rpages_ind; arena_run_t *run; arena_bin_t *bin; - index_t actual_binind; + index_t run_binind, actual_binind; arena_bin_info_t *bin_info; arena_chunk_map_misc_t *miscelm; void *rpages; @@ -774,9 +771,10 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) pageind); miscelm = arena_miscelm_get(chunk, rpages_ind); run = &miscelm->run; - bin = run->bin; + run_binind = run->binind; + bin = &arena->bins[run_binind]; actual_binind = bin - arena->bins; - assert(binind == actual_binind); + assert(run_binind == actual_binind); bin_info = &arena_bin_info[actual_binind]; rpages = arena_miscelm_to_rpages(miscelm); assert(((uintptr_t)ptr - ((uintptr_t)rpages + diff --git a/src/arena.c b/src/arena.c index bbe58fa6..8872331d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -155,9 +155,6 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) ret = (void *)((uintptr_t)rpages + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(bin_info->reg_interval * regind)); run->nfree--; - if (regind == run->nextind) - run->nextind++; - assert(regind < run->nextind); return (ret); } @@ -361,26 +358,12 @@ arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); - /* - * Propagate the dirty and unzeroed flags to the allocated small run, - * so that arena_dalloc_bin_run() has the ability to conditionally trim - * clean pages. - */ - arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); - if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, - run_ind) == 0) - arena_run_page_validate_zeroed(chunk, run_ind); - for (i = 1; i < need_pages - 1; i++) { + for (i = 0; i < need_pages; i++) { arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) arena_run_page_validate_zeroed(chunk, run_ind+i); } - arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, - binind, flag_dirty); - if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, - run_ind+need_pages-1) == 0) - arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } @@ -1002,8 +985,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_mapbits_large_size_get(chunk, run_ind+(size>>LG_PAGE)-1) == 0); } else { - index_t binind = arena_bin_index(arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; + arena_bin_info_t *bin_info = &arena_bin_info[run->binind]; size = bin_info->run_size; } run_pages = (size >> LG_PAGE); @@ -1199,8 +1181,7 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) run = arena_run_alloc_small(arena, bin_info->run_size, binind); if (run != NULL) { /* Initialize run internals. */ - run->bin = bin; - run->nextind = 0; + run->binind = binind; run->nfree = bin_info->nregs; bitmap_init(run->bitmap, &bin_info->bitmap_info); } @@ -1652,54 +1633,15 @@ static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_bin_t *bin) { - index_t binind; - arena_bin_info_t *bin_info; - size_t npages, run_ind, past; - arena_chunk_map_misc_t *miscelm; - void *rpages; assert(run != bin->runcur); assert(arena_run_tree_search(&bin->runs, arena_run_to_miscelm(run)) == NULL); - binind = arena_bin_index(chunk->arena, run->bin); - bin_info = &arena_bin_info[binind]; - malloc_mutex_unlock(&bin->lock); /******************************/ - npages = bin_info->run_size >> LG_PAGE; - miscelm = arena_run_to_miscelm(run); - run_ind = arena_miscelm_to_pageind(miscelm); - rpages = arena_miscelm_to_rpages(miscelm); - past = (size_t)(PAGE_CEILING((uintptr_t)rpages + - (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * - bin_info->reg_interval - bin_info->redzone_size) - - (uintptr_t)chunk) >> LG_PAGE); malloc_mutex_lock(&arena->lock); - - /* - * If the run was originally clean, and some pages were never touched, - * trim the clean pages before deallocating the dirty portion of the - * run. - */ - assert(arena_mapbits_dirty_get(chunk, run_ind) == - arena_mapbits_dirty_get(chunk, run_ind+npages-1)); - if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < - npages) { - /* Trim clean pages. Convert to large run beforehand. */ - assert(npages > 0); - if (past > run_ind) { - arena_mapbits_large_set(chunk, run_ind, - bin_info->run_size, 0); - arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); - arena_run_trim_tail(arena, chunk, run, (npages << - LG_PAGE), ((past - run_ind) << LG_PAGE), false); - arena_run_dalloc(arena, run, true, false); - } else - arena_run_dalloc(arena, run, false, false); - /* npages = past - run_ind; */ - } else - arena_run_dalloc(arena, run, true, false); + arena_run_dalloc(arena, run, true, false); malloc_mutex_unlock(&arena->lock); /****************************/ malloc_mutex_lock(&bin->lock); @@ -1742,9 +1684,8 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); run = &arena_miscelm_get(chunk, rpages_ind)->run; - bin = run->bin; - binind = arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, - pageind)); + binind = run->binind; + bin = &arena->bins[binind]; bin_info = &arena_bin_info[binind]; if (config_fill || config_stats) size = bin_info->reg_size; @@ -1783,7 +1724,7 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, rpages_ind = pageind - arena_mapbits_small_runind_get(chunk, pageind); run = &arena_miscelm_get(chunk, rpages_ind)->run; - bin = run->bin; + bin = &arena->bins[run->binind]; malloc_mutex_lock(&bin->lock); arena_dalloc_bin_locked_impl(arena, chunk, ptr, bitselm, false); malloc_mutex_unlock(&bin->lock); From 44c97b712ef1669a4c75ea97e8d47c0535e9ec71 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 12 Oct 2014 13:03:20 -0700 Subject: [PATCH 152/352] Fix a prof_tctx_t/prof_tdata_t cleanup race. Fix a prof_tctx_t/prof_tdata_t cleanup race by storing a copy of thr_uid in prof_tctx_t, so that the associated tdata need not be present during tctx teardown. --- include/jemalloc/internal/prof.h | 6 ++++++ src/prof.c | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index c8014717..5103146b 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -89,6 +89,12 @@ struct prof_tctx_s { /* Thread data for thread that performed the allocation. */ prof_tdata_t *tdata; + /* + * Copy of tdata->thr_uid, necessary because tdata may be defunct during + * teardown. + */ + uint64_t thr_uid; + /* Profiling counters, protected by tdata->lock. */ prof_cnt_t cnts; diff --git a/src/prof.c b/src/prof.c index 3e2e4277..40163271 100644 --- a/src/prof.c +++ b/src/prof.c @@ -128,8 +128,8 @@ static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name); JEMALLOC_INLINE_C int prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) { - uint64_t a_uid = a->tdata->thr_uid; - uint64_t b_uid = b->tdata->thr_uid; + uint64_t a_uid = a->thr_uid; + uint64_t b_uid = b->thr_uid; return ((a_uid > b_uid) - (a_uid < b_uid)); } @@ -755,6 +755,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) return (NULL); } ret.p->tdata = tdata; + ret.p->thr_uid = tdata->thr_uid; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); ret.p->gctx = gctx; ret.p->prepared = true; @@ -1051,9 +1052,8 @@ prof_tctx_dump_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) if (prof_dump_printf(propagate_err, " t%"PRIu64": %"PRIu64": %"PRIu64" [%"PRIu64": %"PRIu64"]\n", - tctx->tdata->thr_uid, tctx->dump_cnts.curobjs, - tctx->dump_cnts.curbytes, tctx->dump_cnts.accumobjs, - tctx->dump_cnts.accumbytes)) + tctx->thr_uid, tctx->dump_cnts.curobjs, tctx->dump_cnts.curbytes, + tctx->dump_cnts.accumobjs, tctx->dump_cnts.accumbytes)) return (tctx); return (NULL); } From 3c4d92e82a31f652a7c77ca937a02d0185085b06 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 12 Oct 2014 22:53:59 -0700 Subject: [PATCH 153/352] Add per size class huge allocation statistics. Add per size class huge allocation statistics, and normalize various stats: - Change the arenas.nlruns type from size_t to unsigned. - Add the arenas.nhchunks and arenas.hchunks..size mallctl's. - Replace the stats.arenas..bins..allocated mallctl with stats.arenas..bins..curregs . - Add the stats.arenas..hchunks..nmalloc, stats.arenas..hchunks..ndalloc, stats.arenas..hchunks..nrequests, and stats.arenas..hchunks..curhchunks mallctl's. --- doc/jemalloc.xml.in | 98 +++++++-- include/jemalloc/internal/arena.h | 9 +- include/jemalloc/internal/ctl.h | 1 + include/jemalloc/internal/stats.h | 34 ++- src/arena.c | 79 ++++--- src/ctl.c | 334 +++++++++++++++++++----------- src/huge.c | 168 +++++++++------ src/stats.c | 215 +++++++++++-------- test/unit/mallctl.c | 21 +- test/unit/stats.c | 103 ++++++++- 10 files changed, 724 insertions(+), 338 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 7da1498a..8111fc1d 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -406,11 +406,12 @@ for (i = 0; i < nbins; i++) { functions simultaneously. If is specified during configuration, “m” and “a” can be specified to omit merged arena and per arena statistics, respectively; - “b” and “l” can be specified to omit per size - class statistics for bins and large objects, respectively. Unrecognized - characters are silently ignored. Note that thread caching may prevent - some statistics from being completely up to date, since extra locking - would be required to merge counters that track thread cache operations. + “b”, “l”, and “h” can be specified to + omit per size class statistics for bins, large objects, and huge objects, + respectively. Unrecognized characters are silently ignored. Note that + thread caching may prevent some statistics from being completely up to + date, since extra locking would be required to merge counters that track + thread cache operations. The malloc_usable_size function @@ -1520,7 +1521,7 @@ malloc_conf = "xmalloc:true";]]> arenas.nlruns - (size_t) + (unsigned) r- Total number of large size classes. @@ -1536,6 +1537,25 @@ malloc_conf = "xmalloc:true";]]> class. + + + arenas.nhchunks + (unsigned) + r- + + Total number of huge size classes. + + + + + arenas.hchunks.<i>.size + (size_t) + r- + + Maximum size supported by this huge size + class. + + arenas.extend @@ -1945,17 +1965,6 @@ malloc_conf = "xmalloc:true";]]> - - - stats.arenas.<i>.bins.<j>.allocated - (size_t) - r- - [] - - Current number of bytes allocated by - bin. - - stats.arenas.<i>.bins.<j>.nmalloc @@ -1989,6 +1998,17 @@ malloc_conf = "xmalloc:true";]]> requests. + + + stats.arenas.<i>.bins.<j>.curregs + (size_t) + r- + [] + + Current number of regions for this size + class. + + stats.arenas.<i>.bins.<j>.nfills @@ -2083,6 +2103,50 @@ malloc_conf = "xmalloc:true";]]> Current number of runs for this size class. + + + + stats.arenas.<i>.hchunks.<j>.nmalloc + (uint64_t) + r- + [] + + Cumulative number of allocation requests for this size + class served directly by the arena. + + + + + stats.arenas.<i>.hchunks.<j>.ndalloc + (uint64_t) + r- + [] + + Cumulative number of deallocation requests for this + size class served directly by the arena. + + + + + stats.arenas.<i>.hchunks.<j>.nrequests + (uint64_t) + r- + [] + + Cumulative number of allocation requests for this size + class. + + + + + stats.arenas.<i>.hchunks.<j>.curhchunks + (size_t) + r- + [] + + Current number of huge allocations for this size class. + + diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 28ff7271..c31c8d7d 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -335,11 +335,12 @@ extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t map_misc_offset; extern size_t arena_maxrun; /* Max run size for arenas. */ extern size_t arena_maxclass; /* Max size class for arenas. */ -extern size_t nlclasses; /* Number of large size classes. */ +extern unsigned nlclasses; /* Number of large size classes. */ +extern unsigned nhclasses; /* Number of huge size classes. */ -void *arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t size, +void *arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t usize, size_t alignment, bool *zero); -void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size); +void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize); void arena_purge_all(arena_t *arena); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, uint64_t prof_accumbytes); @@ -387,7 +388,7 @@ dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats); + malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats); arena_t *arena_new(unsigned ind); void arena_boot(void); void arena_prefork(arena_t *arena); diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index 2d301bf1..a3e899ea 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -46,6 +46,7 @@ struct ctl_arena_stats_s { malloc_bin_stats_t bstats[NBINS]; malloc_large_stats_t *lstats; /* nlclasses elements. */ + malloc_huge_stats_t *hstats; /* nhclasses elements. */ }; struct ctl_stats_s { diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index 6104cb3a..d8600ed4 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -4,6 +4,7 @@ typedef struct tcache_bin_stats_s tcache_bin_stats_t; typedef struct malloc_bin_stats_s malloc_bin_stats_t; typedef struct malloc_large_stats_s malloc_large_stats_t; +typedef struct malloc_huge_stats_s malloc_huge_stats_t; typedef struct arena_stats_s arena_stats_t; typedef struct chunk_stats_s chunk_stats_t; @@ -20,12 +21,6 @@ struct tcache_bin_stats_s { }; struct malloc_bin_stats_s { - /* - * Current number of bytes allocated, including objects currently - * cached by tcache. - */ - size_t allocated; - /* * Total number of allocation/deallocation requests served directly by * the bin. Note that tcache may allocate an object, then recycle it @@ -42,6 +37,12 @@ struct malloc_bin_stats_s { */ uint64_t nrequests; + /* + * Current number of regions of this size class, including regions + * currently cached by tcache. + */ + size_t curregs; + /* Number of tcache fills from this bin. */ uint64_t nfills; @@ -78,10 +79,25 @@ struct malloc_large_stats_s { */ uint64_t nrequests; - /* Current number of runs of this size class. */ + /* + * Current number of runs of this size class, including runs currently + * cached by tcache. + */ size_t curruns; }; +struct malloc_huge_stats_s { + /* + * Total number of allocation/deallocation requests served directly by + * the arena. + */ + uint64_t nmalloc; + uint64_t ndalloc; + + /* Current number of (multi-)chunk allocations of this size class. */ + size_t curhchunks; +}; + struct arena_stats_s { /* Number of bytes currently mapped. */ size_t mapped; @@ -104,10 +120,12 @@ struct arena_stats_s { size_t allocated_huge; uint64_t nmalloc_huge; uint64_t ndalloc_huge; - uint64_t nrequests_huge; /* One element for each large size class. */ malloc_large_stats_t *lstats; + + /* One element for each huge size class. */ + malloc_huge_stats_t *hstats; }; struct chunk_stats_s { diff --git a/src/arena.c b/src/arena.c index 8872331d..74c36323 100644 --- a/src/arena.c +++ b/src/arena.c @@ -11,7 +11,8 @@ size_t map_bias; size_t map_misc_offset; size_t arena_maxrun; /* Max run size for arenas. */ size_t arena_maxclass; /* Max size class for arenas. */ -size_t nlclasses; /* Number of large size classes. */ +unsigned nlclasses; /* Number of large size classes. */ +unsigned nhclasses; /* Number of huge size classes. */ /******************************************************************************/ /* @@ -411,7 +412,7 @@ arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, } void * -arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t size, +arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t usize, size_t alignment, bool *zero) { void *ret; @@ -422,26 +423,33 @@ arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t size, chunk_alloc = arena->chunk_alloc; chunk_dalloc = arena->chunk_dalloc; if (config_stats) { + index_t index = size2index(usize) - nlclasses - NBINS; + /* Optimistically update stats prior to unlocking. */ - arena->stats.mapped += size; - arena->stats.allocated_huge += size; + arena->stats.allocated_huge += usize; arena->stats.nmalloc_huge++; - arena->stats.nrequests_huge++; + arena->stats.hstats[index].nmalloc++; + arena->stats.hstats[index].curhchunks++; + arena->stats.mapped += usize; } - arena->nactive += (size >> LG_PAGE); + arena->nactive += (usize >> LG_PAGE); malloc_mutex_unlock(&arena->lock); ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, - new_addr, size, alignment, zero); + new_addr, usize, alignment, zero); if (config_stats) { if (ret != NULL) - stats_cactive_add(size); + stats_cactive_add(usize); else { - /* Revert optimistic stats updates. */ + index_t index = size2index(usize) - nlclasses - NBINS; + malloc_mutex_lock(&arena->lock); - arena->stats.mapped -= size; - arena->stats.allocated_huge -= size; + /* Revert optimistic stats updates. */ + arena->stats.allocated_huge -= usize; arena->stats.nmalloc_huge--; + arena->stats.hstats[index].nmalloc--; + arena->stats.hstats[index].curhchunks--; + arena->stats.mapped -= usize; malloc_mutex_unlock(&arena->lock); } } @@ -534,21 +542,25 @@ arena_chunk_dalloc_internal(arena_t *arena, arena_chunk_t *chunk) } void -arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t size) +arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) { chunk_dalloc_t *chunk_dalloc; malloc_mutex_lock(&arena->lock); chunk_dalloc = arena->chunk_dalloc; if (config_stats) { - arena->stats.mapped -= size; - arena->stats.allocated_huge -= size; + index_t index = size2index(usize) - nlclasses - NBINS; + arena->stats.ndalloc_huge++; - stats_cactive_sub(size); + arena->stats.allocated_huge -= usize; + arena->stats.hstats[index].ndalloc++; + arena->stats.hstats[index].curhchunks--; + arena->stats.mapped -= usize; + stats_cactive_sub(usize); } - arena->nactive -= (size >> LG_PAGE); + arena->nactive -= (usize >> LG_PAGE); malloc_mutex_unlock(&arena->lock); - chunk_dalloc(chunk, size, arena->ind); + chunk_dalloc(chunk, usize, arena->ind); } static void @@ -1300,9 +1312,9 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, tbin->avail[nfill - 1 - i] = ptr; } if (config_stats) { - bin->stats.allocated += i * arena_bin_info[binind].reg_size; bin->stats.nmalloc += i; bin->stats.nrequests += tbin->tstats.nrequests; + bin->stats.curregs += i; bin->stats.nfills++; tbin->tstats.nrequests = 0; } @@ -1436,9 +1448,9 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) } if (config_stats) { - bin->stats.allocated += size; bin->stats.nmalloc++; bin->stats.nrequests++; + bin->stats.curregs++; } malloc_mutex_unlock(&bin->lock); if (config_prof && !isthreaded && arena_prof_accum(arena, size)) @@ -1678,7 +1690,6 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_run_t *run; arena_bin_t *bin; arena_bin_info_t *bin_info; - size_t size; index_t binind; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; @@ -1687,8 +1698,6 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, binind = run->binind; bin = &arena->bins[binind]; bin_info = &arena_bin_info[binind]; - if (config_fill || config_stats) - size = bin_info->reg_size; if (!junked && config_fill && unlikely(opt_junk)) arena_dalloc_junk_small(ptr, bin_info); @@ -1701,8 +1710,8 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_bin_lower_run(arena, chunk, run, bin); if (config_stats) { - bin->stats.allocated -= size; bin->stats.ndalloc++; + bin->stats.curregs--; } } @@ -2102,7 +2111,7 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats) + malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats) { unsigned i; @@ -2122,7 +2131,6 @@ arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, astats->allocated_huge += arena->stats.allocated_huge; astats->nmalloc_huge += arena->stats.nmalloc_huge; astats->ndalloc_huge += arena->stats.ndalloc_huge; - astats->nrequests_huge += arena->stats.nrequests_huge; for (i = 0; i < nlclasses; i++) { lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; @@ -2130,16 +2138,22 @@ arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, lstats[i].nrequests += arena->stats.lstats[i].nrequests; lstats[i].curruns += arena->stats.lstats[i].curruns; } + + for (i = 0; i < nhclasses; i++) { + hstats[i].nmalloc += arena->stats.hstats[i].nmalloc; + hstats[i].ndalloc += arena->stats.hstats[i].ndalloc; + hstats[i].curhchunks += arena->stats.hstats[i].curhchunks; + } malloc_mutex_unlock(&arena->lock); for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(&bin->lock); - bstats[i].allocated += bin->stats.allocated; bstats[i].nmalloc += bin->stats.nmalloc; bstats[i].ndalloc += bin->stats.ndalloc; bstats[i].nrequests += bin->stats.nrequests; + bstats[i].curregs += bin->stats.curregs; if (config_tcache) { bstats[i].nfills += bin->stats.nfills; bstats[i].nflushes += bin->stats.nflushes; @@ -2159,12 +2173,13 @@ arena_new(unsigned ind) arena_bin_t *bin; /* - * Allocate arena and arena->lstats contiguously, mainly because there - * is no way to clean up if base_alloc() OOMs. + * Allocate arena, arena->lstats, and arena->hstats contiguously, mainly + * because there is no way to clean up if base_alloc() OOMs. */ if (config_stats) { arena = (arena_t *)base_alloc(CACHELINE_CEILING(sizeof(arena_t)) - + nlclasses * sizeof(malloc_large_stats_t)); + + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t) + + nhclasses) * sizeof(malloc_huge_stats_t)); } else arena = (arena_t *)base_alloc(sizeof(arena_t)); if (arena == NULL) @@ -2184,6 +2199,11 @@ arena_new(unsigned ind) CACHELINE_CEILING(sizeof(arena_t))); memset(arena->stats.lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); + arena->stats.hstats = (malloc_huge_stats_t *)(((void *)arena) + + CACHELINE_CEILING(sizeof(arena_t)) + + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t))); + memset(arena->stats.hstats, 0, nhclasses * + sizeof(malloc_huge_stats_t)); if (config_tcache) ql_new(&arena->tcache_ql); } @@ -2369,6 +2389,7 @@ arena_boot(void) } assert(arena_maxclass > 0); nlclasses = size2index(arena_maxclass) - size2index(SMALL_MAXCLASS); + nhclasses = NSIZES - nlclasses - NBINS; bin_info_init(); } diff --git a/src/ctl.c b/src/ctl.c index 37f8f42a..72598b3d 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -122,6 +122,8 @@ CTL_PROTO(arenas_bin_i_run_size) INDEX_PROTO(arenas_bin_i) CTL_PROTO(arenas_lrun_i_size) INDEX_PROTO(arenas_lrun_i) +CTL_PROTO(arenas_hchunk_i_size) +INDEX_PROTO(arenas_hchunk_i) CTL_PROTO(arenas_narenas) CTL_PROTO(arenas_initialized) CTL_PROTO(arenas_quantum) @@ -130,6 +132,7 @@ CTL_PROTO(arenas_tcache_max) CTL_PROTO(arenas_nbins) CTL_PROTO(arenas_nhbins) CTL_PROTO(arenas_nlruns) +CTL_PROTO(arenas_nhchunks) CTL_PROTO(arenas_extend) CTL_PROTO(prof_thread_active_init) CTL_PROTO(prof_active) @@ -152,10 +155,10 @@ CTL_PROTO(stats_arenas_i_huge_allocated) CTL_PROTO(stats_arenas_i_huge_nmalloc) CTL_PROTO(stats_arenas_i_huge_ndalloc) CTL_PROTO(stats_arenas_i_huge_nrequests) -CTL_PROTO(stats_arenas_i_bins_j_allocated) CTL_PROTO(stats_arenas_i_bins_j_nmalloc) CTL_PROTO(stats_arenas_i_bins_j_ndalloc) CTL_PROTO(stats_arenas_i_bins_j_nrequests) +CTL_PROTO(stats_arenas_i_bins_j_curregs) CTL_PROTO(stats_arenas_i_bins_j_nfills) CTL_PROTO(stats_arenas_i_bins_j_nflushes) CTL_PROTO(stats_arenas_i_bins_j_nruns) @@ -167,6 +170,11 @@ CTL_PROTO(stats_arenas_i_lruns_j_ndalloc) CTL_PROTO(stats_arenas_i_lruns_j_nrequests) CTL_PROTO(stats_arenas_i_lruns_j_curruns) INDEX_PROTO(stats_arenas_i_lruns_j) +CTL_PROTO(stats_arenas_i_hchunks_j_nmalloc) +CTL_PROTO(stats_arenas_i_hchunks_j_ndalloc) +CTL_PROTO(stats_arenas_i_hchunks_j_nrequests) +CTL_PROTO(stats_arenas_i_hchunks_j_curhchunks) +INDEX_PROTO(stats_arenas_i_hchunks_j) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_dss) CTL_PROTO(stats_arenas_i_pactive) @@ -221,60 +229,60 @@ static const ctl_named_node_t thread_node[] = { }; static const ctl_named_node_t config_node[] = { - {NAME("debug"), CTL(config_debug)}, - {NAME("fill"), CTL(config_fill)}, - {NAME("lazy_lock"), CTL(config_lazy_lock)}, - {NAME("munmap"), CTL(config_munmap)}, - {NAME("prof"), CTL(config_prof)}, - {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, - {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, - {NAME("stats"), CTL(config_stats)}, - {NAME("tcache"), CTL(config_tcache)}, - {NAME("tls"), CTL(config_tls)}, - {NAME("utrace"), CTL(config_utrace)}, - {NAME("valgrind"), CTL(config_valgrind)}, - {NAME("xmalloc"), CTL(config_xmalloc)} + {NAME("debug"), CTL(config_debug)}, + {NAME("fill"), CTL(config_fill)}, + {NAME("lazy_lock"), CTL(config_lazy_lock)}, + {NAME("munmap"), CTL(config_munmap)}, + {NAME("prof"), CTL(config_prof)}, + {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, + {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, + {NAME("stats"), CTL(config_stats)}, + {NAME("tcache"), CTL(config_tcache)}, + {NAME("tls"), CTL(config_tls)}, + {NAME("utrace"), CTL(config_utrace)}, + {NAME("valgrind"), CTL(config_valgrind)}, + {NAME("xmalloc"), CTL(config_xmalloc)} }; static const ctl_named_node_t opt_node[] = { - {NAME("abort"), CTL(opt_abort)}, - {NAME("dss"), CTL(opt_dss)}, - {NAME("lg_chunk"), CTL(opt_lg_chunk)}, - {NAME("narenas"), CTL(opt_narenas)}, - {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, - {NAME("stats_print"), CTL(opt_stats_print)}, - {NAME("junk"), CTL(opt_junk)}, - {NAME("zero"), CTL(opt_zero)}, - {NAME("quarantine"), CTL(opt_quarantine)}, - {NAME("redzone"), CTL(opt_redzone)}, - {NAME("utrace"), CTL(opt_utrace)}, - {NAME("xmalloc"), CTL(opt_xmalloc)}, - {NAME("tcache"), CTL(opt_tcache)}, - {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, - {NAME("prof"), CTL(opt_prof)}, - {NAME("prof_prefix"), CTL(opt_prof_prefix)}, - {NAME("prof_active"), CTL(opt_prof_active)}, + {NAME("abort"), CTL(opt_abort)}, + {NAME("dss"), CTL(opt_dss)}, + {NAME("lg_chunk"), CTL(opt_lg_chunk)}, + {NAME("narenas"), CTL(opt_narenas)}, + {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, + {NAME("stats_print"), CTL(opt_stats_print)}, + {NAME("junk"), CTL(opt_junk)}, + {NAME("zero"), CTL(opt_zero)}, + {NAME("quarantine"), CTL(opt_quarantine)}, + {NAME("redzone"), CTL(opt_redzone)}, + {NAME("utrace"), CTL(opt_utrace)}, + {NAME("xmalloc"), CTL(opt_xmalloc)}, + {NAME("tcache"), CTL(opt_tcache)}, + {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, + {NAME("prof"), CTL(opt_prof)}, + {NAME("prof_prefix"), CTL(opt_prof_prefix)}, + {NAME("prof_active"), CTL(opt_prof_active)}, {NAME("prof_thread_active_init"), CTL(opt_prof_thread_active_init)}, - {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, - {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, - {NAME("prof_gdump"), CTL(opt_prof_gdump)}, - {NAME("prof_final"), CTL(opt_prof_final)}, - {NAME("prof_leak"), CTL(opt_prof_leak)}, - {NAME("prof_accum"), CTL(opt_prof_accum)} + {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, + {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, + {NAME("prof_gdump"), CTL(opt_prof_gdump)}, + {NAME("prof_final"), CTL(opt_prof_final)}, + {NAME("prof_leak"), CTL(opt_prof_leak)}, + {NAME("prof_accum"), CTL(opt_prof_accum)} }; static const ctl_named_node_t chunk_node[] = { - {NAME("alloc"), CTL(arena_i_chunk_alloc)}, - {NAME("dalloc"), CTL(arena_i_chunk_dalloc)} + {NAME("alloc"), CTL(arena_i_chunk_alloc)}, + {NAME("dalloc"), CTL(arena_i_chunk_dalloc)} }; static const ctl_named_node_t arena_i_node[] = { - {NAME("purge"), CTL(arena_i_purge)}, - {NAME("dss"), CTL(arena_i_dss)}, - {NAME("chunk"), CHILD(named, chunk)}, + {NAME("purge"), CTL(arena_i_purge)}, + {NAME("dss"), CTL(arena_i_dss)}, + {NAME("chunk"), CHILD(named, chunk)}, }; static const ctl_named_node_t super_arena_i_node[] = { - {NAME(""), CHILD(named, arena_i)} + {NAME(""), CHILD(named, arena_i)} }; static const ctl_indexed_node_t arena_node[] = { @@ -282,12 +290,12 @@ static const ctl_indexed_node_t arena_node[] = { }; static const ctl_named_node_t arenas_bin_i_node[] = { - {NAME("size"), CTL(arenas_bin_i_size)}, - {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("run_size"), CTL(arenas_bin_i_run_size)} + {NAME("size"), CTL(arenas_bin_i_size)}, + {NAME("nregs"), CTL(arenas_bin_i_nregs)}, + {NAME("run_size"), CTL(arenas_bin_i_run_size)} }; static const ctl_named_node_t super_arenas_bin_i_node[] = { - {NAME(""), CHILD(named, arenas_bin_i)} + {NAME(""), CHILD(named, arenas_bin_i)} }; static const ctl_indexed_node_t arenas_bin_node[] = { @@ -295,28 +303,41 @@ static const ctl_indexed_node_t arenas_bin_node[] = { }; static const ctl_named_node_t arenas_lrun_i_node[] = { - {NAME("size"), CTL(arenas_lrun_i_size)} + {NAME("size"), CTL(arenas_lrun_i_size)} }; static const ctl_named_node_t super_arenas_lrun_i_node[] = { - {NAME(""), CHILD(named, arenas_lrun_i)} + {NAME(""), CHILD(named, arenas_lrun_i)} }; static const ctl_indexed_node_t arenas_lrun_node[] = { {INDEX(arenas_lrun_i)} }; +static const ctl_named_node_t arenas_hchunk_i_node[] = { + {NAME("size"), CTL(arenas_hchunk_i_size)} +}; +static const ctl_named_node_t super_arenas_hchunk_i_node[] = { + {NAME(""), CHILD(named, arenas_hchunk_i)} +}; + +static const ctl_indexed_node_t arenas_hchunk_node[] = { + {INDEX(arenas_hchunk_i)} +}; + static const ctl_named_node_t arenas_node[] = { - {NAME("narenas"), CTL(arenas_narenas)}, - {NAME("initialized"), CTL(arenas_initialized)}, - {NAME("quantum"), CTL(arenas_quantum)}, - {NAME("page"), CTL(arenas_page)}, - {NAME("tcache_max"), CTL(arenas_tcache_max)}, - {NAME("nbins"), CTL(arenas_nbins)}, - {NAME("nhbins"), CTL(arenas_nhbins)}, - {NAME("bin"), CHILD(indexed, arenas_bin)}, - {NAME("nlruns"), CTL(arenas_nlruns)}, - {NAME("lrun"), CHILD(indexed, arenas_lrun)}, - {NAME("extend"), CTL(arenas_extend)} + {NAME("narenas"), CTL(arenas_narenas)}, + {NAME("initialized"), CTL(arenas_initialized)}, + {NAME("quantum"), CTL(arenas_quantum)}, + {NAME("page"), CTL(arenas_page)}, + {NAME("tcache_max"), CTL(arenas_tcache_max)}, + {NAME("nbins"), CTL(arenas_nbins)}, + {NAME("nhbins"), CTL(arenas_nhbins)}, + {NAME("bin"), CHILD(indexed, arenas_bin)}, + {NAME("nlruns"), CTL(arenas_nlruns)}, + {NAME("lrun"), CHILD(indexed, arenas_lrun)}, + {NAME("nhchunks"), CTL(arenas_nhchunks)}, + {NAME("hchunk"), CHILD(indexed, arenas_hchunk)}, + {NAME("extend"), CTL(arenas_extend)} }; static const ctl_named_node_t prof_node[] = { @@ -329,45 +350,45 @@ static const ctl_named_node_t prof_node[] = { }; static const ctl_named_node_t stats_chunks_node[] = { - {NAME("current"), CTL(stats_chunks_current)}, - {NAME("total"), CTL(stats_chunks_total)}, - {NAME("high"), CTL(stats_chunks_high)} + {NAME("current"), CTL(stats_chunks_current)}, + {NAME("total"), CTL(stats_chunks_total)}, + {NAME("high"), CTL(stats_chunks_high)} }; static const ctl_named_node_t stats_arenas_i_small_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} + {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} }; static const ctl_named_node_t stats_arenas_i_large_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} + {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} }; static const ctl_named_node_t stats_arenas_i_huge_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_huge_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_huge_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_huge_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_huge_nrequests)}, + {NAME("allocated"), CTL(stats_arenas_i_huge_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_huge_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_huge_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_huge_nrequests)} }; static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, - {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, - {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, - {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, - {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, - {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} + {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, + {NAME("curregs"), CTL(stats_arenas_i_bins_j_curregs)}, + {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, + {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, + {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, + {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, + {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} }; static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { - {NAME(""), CHILD(named, stats_arenas_i_bins_j)} + {NAME(""), CHILD(named, stats_arenas_i_bins_j)} }; static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { @@ -375,36 +396,51 @@ static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { }; static const ctl_named_node_t stats_arenas_i_lruns_j_node[] = { - {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, - {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} + {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, + {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} }; static const ctl_named_node_t super_stats_arenas_i_lruns_j_node[] = { - {NAME(""), CHILD(named, stats_arenas_i_lruns_j)} + {NAME(""), CHILD(named, stats_arenas_i_lruns_j)} }; static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = { {INDEX(stats_arenas_i_lruns_j)} }; +static const ctl_named_node_t stats_arenas_i_hchunks_j_node[] = { + {NAME("nmalloc"), CTL(stats_arenas_i_hchunks_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_hchunks_j_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_hchunks_j_nrequests)}, + {NAME("curhchunks"), CTL(stats_arenas_i_hchunks_j_curhchunks)} +}; +static const ctl_named_node_t super_stats_arenas_i_hchunks_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_hchunks_j)} +}; + +static const ctl_indexed_node_t stats_arenas_i_hchunks_node[] = { + {INDEX(stats_arenas_i_hchunks_j)} +}; + static const ctl_named_node_t stats_arenas_i_node[] = { - {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, - {NAME("dss"), CTL(stats_arenas_i_dss)}, - {NAME("pactive"), CTL(stats_arenas_i_pactive)}, - {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, - {NAME("mapped"), CTL(stats_arenas_i_mapped)}, - {NAME("npurge"), CTL(stats_arenas_i_npurge)}, - {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, - {NAME("purged"), CTL(stats_arenas_i_purged)}, - {NAME("small"), CHILD(named, stats_arenas_i_small)}, - {NAME("large"), CHILD(named, stats_arenas_i_large)}, - {NAME("huge"), CHILD(named, stats_arenas_i_huge)}, - {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, - {NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)} + {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, + {NAME("dss"), CTL(stats_arenas_i_dss)}, + {NAME("pactive"), CTL(stats_arenas_i_pactive)}, + {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, + {NAME("mapped"), CTL(stats_arenas_i_mapped)}, + {NAME("npurge"), CTL(stats_arenas_i_npurge)}, + {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, + {NAME("purged"), CTL(stats_arenas_i_purged)}, + {NAME("small"), CHILD(named, stats_arenas_i_small)}, + {NAME("large"), CHILD(named, stats_arenas_i_large)}, + {NAME("huge"), CHILD(named, stats_arenas_i_huge)}, + {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, + {NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)}, + {NAME("hchunks"), CHILD(indexed, stats_arenas_i_hchunks)} }; static const ctl_named_node_t super_stats_arenas_i_node[] = { - {NAME(""), CHILD(named, stats_arenas_i)} + {NAME(""), CHILD(named, stats_arenas_i)} }; static const ctl_indexed_node_t stats_arenas_node[] = { @@ -412,12 +448,12 @@ static const ctl_indexed_node_t stats_arenas_node[] = { }; static const ctl_named_node_t stats_node[] = { - {NAME("cactive"), CTL(stats_cactive)}, - {NAME("allocated"), CTL(stats_allocated)}, - {NAME("active"), CTL(stats_active)}, - {NAME("mapped"), CTL(stats_mapped)}, - {NAME("chunks"), CHILD(named, stats_chunks)}, - {NAME("arenas"), CHILD(indexed, stats_arenas)} + {NAME("cactive"), CTL(stats_cactive)}, + {NAME("allocated"), CTL(stats_allocated)}, + {NAME("active"), CTL(stats_active)}, + {NAME("mapped"), CTL(stats_mapped)}, + {NAME("chunks"), CHILD(named, stats_chunks)}, + {NAME("arenas"), CHILD(indexed, stats_arenas)} }; static const ctl_named_node_t root_node[] = { @@ -453,6 +489,13 @@ ctl_arena_init(ctl_arena_stats_t *astats) return (true); } + if (astats->hstats == NULL) { + astats->hstats = (malloc_huge_stats_t *)a0malloc(nhclasses * + sizeof(malloc_huge_stats_t)); + if (astats->hstats == NULL) + return (true); + } + return (false); } @@ -472,6 +515,8 @@ ctl_arena_clear(ctl_arena_stats_t *astats) memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t)); memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); + memset(astats->hstats, 0, nhclasses * + sizeof(malloc_huge_stats_t)); } } @@ -481,10 +526,12 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) unsigned i; arena_stats_merge(arena, &cstats->dss, &cstats->pactive, - &cstats->pdirty, &cstats->astats, cstats->bstats, cstats->lstats); + &cstats->pdirty, &cstats->astats, cstats->bstats, cstats->lstats, + cstats->hstats); for (i = 0; i < NBINS; i++) { - cstats->allocated_small += cstats->bstats[i].allocated; + cstats->allocated_small += cstats->bstats[i].curregs * + index2size(i); cstats->nmalloc_small += cstats->bstats[i].nmalloc; cstats->ndalloc_small += cstats->bstats[i].ndalloc; cstats->nrequests_small += cstats->bstats[i].nrequests; @@ -517,20 +564,12 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->astats.allocated_huge += astats->astats.allocated_huge; sstats->astats.nmalloc_huge += astats->astats.nmalloc_huge; sstats->astats.ndalloc_huge += astats->astats.ndalloc_huge; - sstats->astats.nrequests_huge += astats->astats.nrequests_huge; - - for (i = 0; i < nlclasses; i++) { - sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; - sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; - sstats->lstats[i].nrequests += astats->lstats[i].nrequests; - sstats->lstats[i].curruns += astats->lstats[i].curruns; - } for (i = 0; i < NBINS; i++) { - sstats->bstats[i].allocated += astats->bstats[i].allocated; sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; sstats->bstats[i].nrequests += astats->bstats[i].nrequests; + sstats->bstats[i].curregs += astats->bstats[i].curregs; if (config_tcache) { sstats->bstats[i].nfills += astats->bstats[i].nfills; sstats->bstats[i].nflushes += @@ -540,6 +579,19 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->bstats[i].reruns += astats->bstats[i].reruns; sstats->bstats[i].curruns += astats->bstats[i].curruns; } + + for (i = 0; i < nlclasses; i++) { + sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; + sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; + sstats->lstats[i].nrequests += astats->lstats[i].nrequests; + sstats->lstats[i].curruns += astats->lstats[i].curruns; + } + + for (i = 0; i < nhclasses; i++) { + sstats->hstats[i].nmalloc += astats->hstats[i].nmalloc; + sstats->hstats[i].ndalloc += astats->hstats[i].ndalloc; + sstats->hstats[i].curhchunks += astats->hstats[i].curhchunks; + } } static void @@ -692,6 +744,8 @@ ctl_init(void) for (j = 0; j < i; j++) { a0free( ctl_stats.arenas[j].lstats); + a0free( + ctl_stats.arenas[j].hstats); } a0free(ctl_stats.arenas); ctl_stats.arenas = NULL; @@ -1600,7 +1654,7 @@ arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) return (super_arenas_bin_i_node); } -CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t) +CTL_RO_NL_GEN(arenas_nlruns, nlclasses, unsigned) CTL_RO_NL_GEN(arenas_lrun_i_size, index2size(NBINS+mib[2]), size_t) static const ctl_named_node_t * arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) @@ -1611,6 +1665,17 @@ arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) return (super_arenas_lrun_i_node); } +CTL_RO_NL_GEN(arenas_nhchunks, nhclasses, unsigned) +CTL_RO_NL_GEN(arenas_hchunk_i_size, index2size(NBINS+nlclasses+mib[2]), size_t) +static const ctl_named_node_t * +arenas_hchunk_i_index(const size_t *mib, size_t miblen, size_t i) +{ + + if (i > nhclasses) + return (NULL); + return (super_arenas_hchunk_i_node); +} + static int arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) @@ -1784,16 +1849,16 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nmalloc, CTL_RO_CGEN(config_stats, stats_arenas_i_huge_ndalloc, ctl_stats.arenas[mib[2]].astats.ndalloc_huge, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_huge_nrequests, - ctl_stats.arenas[mib[2]].astats.nrequests_huge, uint64_t) + ctl_stats.arenas[mib[2]].astats.nmalloc_huge, uint64_t) /* Intentional. */ -CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_allocated, - ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curregs, + ctl_stats.arenas[mib[2]].bstats[mib[4]].curregs, size_t) CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nfills, ctl_stats.arenas[mib[2]].bstats[mib[4]].nfills, uint64_t) CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nflushes, @@ -1832,6 +1897,25 @@ stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) return (super_stats_arenas_i_lruns_j_node); } +CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nmalloc, + ctl_stats.arenas[mib[2]].hstats[mib[4]].nmalloc, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_ndalloc, + ctl_stats.arenas[mib[2]].hstats[mib[4]].ndalloc, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_nrequests, + ctl_stats.arenas[mib[2]].hstats[mib[4]].nmalloc, /* Intentional. */ + uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_hchunks_j_curhchunks, + ctl_stats.arenas[mib[2]].hstats[mib[4]].curhchunks, size_t) + +static const ctl_named_node_t * +stats_arenas_i_hchunks_j_index(const size_t *mib, size_t miblen, size_t j) +{ + + if (j > nhclasses) + return (NULL); + return (super_stats_arenas_i_hchunks_j_node); +} + static const ctl_named_node_t * stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) { diff --git a/src/huge.c b/src/huge.c index 6c9b97bb..5f46241d 100644 --- a/src/huge.c +++ b/src/huge.c @@ -104,6 +104,101 @@ huge_dalloc_junk(void *ptr, size_t usize) huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); #endif +static void +huge_ralloc_no_move_stats_update(arena_t *arena, size_t oldsize, size_t usize) +{ + index_t oldindex = size2index(oldsize) - nlclasses - NBINS; + index_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.ndalloc_huge++; + arena->stats.allocated_huge -= oldsize; + arena->stats.hstats[oldindex].ndalloc++; + arena->stats.hstats[oldindex].curhchunks--; + + arena->stats.nmalloc_huge++; + arena->stats.allocated_huge += usize; + arena->stats.hstats[index].nmalloc++; + arena->stats.hstats[index].curhchunks++; +} + +static void +huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, + size_t size, size_t extra, bool zero) +{ + size_t usize_next; + extent_node_t *node, key; + arena_t *arena; + + /* Increase usize to incorporate extra. */ + while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize) + usize = usize_next; + + malloc_mutex_lock(&huge_mtx); + + key.addr = ptr; + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + + arena = node->arena; + + /* Update the size of the huge allocation if it changed. */ + if (oldsize != usize) { + assert(node->size != usize); + node->size = usize; + } + + malloc_mutex_unlock(&huge_mtx); + + /* Fill if necessary. */ + if (oldsize < usize) { + if (zero || (config_fill && unlikely(opt_zero))) + memset(ptr + oldsize, 0, usize - oldsize); + else if (config_fill && unlikely(opt_junk)) + memset(ptr + oldsize, 0xa5, usize - oldsize); + } else if (config_fill && unlikely(opt_junk) && oldsize > usize) + memset(ptr + usize, 0x5a, oldsize - usize); + + if (config_stats) + huge_ralloc_no_move_stats_update(arena, oldsize, usize); +} + +static void +huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) +{ + extent_node_t *node, key; + arena_t *arena; + void *excess_addr; + size_t excess_size; + + malloc_mutex_lock(&huge_mtx); + + key.addr = ptr; + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + + arena = node->arena; + + /* Update the size of the huge allocation. */ + node->size = usize; + + malloc_mutex_unlock(&huge_mtx); + + excess_addr = node->addr + CHUNK_CEILING(usize); + excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); + + /* Zap the excess chunks. */ + huge_dalloc_junk(ptr + usize, oldsize - usize); + if (excess_size > 0) + arena_chunk_dalloc_huge(arena, excess_addr, excess_size); + + if (config_stats) + huge_ralloc_no_move_stats_update(arena, oldsize, usize); +} + static bool huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { size_t usize; @@ -131,7 +226,6 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { assert(node != NULL); assert(node->addr == ptr); - /* Find the current arena. */ arena = node->arena; malloc_mutex_unlock(&huge_mtx); @@ -159,6 +253,10 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { else if (unlikely(opt_zero) && !is_zeroed) memset(ptr + oldsize, 0, usize - oldsize); } + + if (config_stats) + huge_ralloc_no_move_stats_update(arena, oldsize, usize); + return (false); } @@ -185,78 +283,20 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, */ if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize) && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { - size_t usize_next; - - /* Increase usize to incorporate extra. */ - while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < - oldsize) - usize = usize_next; - - /* Update the size of the huge allocation if it changed. */ - if (oldsize != usize) { - extent_node_t *node, key; - - malloc_mutex_lock(&huge_mtx); - - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); - - assert(node->size != usize); - node->size = usize; - - malloc_mutex_unlock(&huge_mtx); - - if (oldsize < usize) { - if (zero || (config_fill && - unlikely(opt_zero))) { - memset(ptr + oldsize, 0, usize - - oldsize); - } else if (config_fill && unlikely(opt_junk)) { - memset(ptr + oldsize, 0xa5, usize - - oldsize); - } - } else if (config_fill && unlikely(opt_junk) && oldsize - > usize) - memset(ptr + usize, 0x5a, oldsize - usize); - } + huge_ralloc_no_move_similar(ptr, oldsize, usize, size, extra, + zero); return (false); } /* Shrink the allocation in-place. */ if (CHUNK_CEILING(oldsize) >= CHUNK_CEILING(usize)) { - extent_node_t *node, key; - void *excess_addr; - size_t excess_size; - - malloc_mutex_lock(&huge_mtx); - - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); - - /* Update the size of the huge allocation. */ - node->size = usize; - - malloc_mutex_unlock(&huge_mtx); - - excess_addr = node->addr + CHUNK_CEILING(usize); - excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); - - /* Zap the excess chunks. */ - huge_dalloc_junk(ptr + usize, oldsize - usize); - if (excess_size > 0) { - arena_chunk_dalloc_huge(node->arena, excess_addr, - excess_size); - } - + huge_ralloc_no_move_shrink(ptr, oldsize, usize); return (false); } /* Attempt to expand the allocation in-place. */ - if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, zero)) { + if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, + zero)) { if (extra == 0) return (true); diff --git a/src/stats.c b/src/stats.c index 5c3d7017..16a18c50 100644 --- a/src/stats.c +++ b/src/stats.c @@ -48,8 +48,10 @@ static void stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); +static void stats_arena_hchunks_print( + void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); static void stats_arena_print(void (*write_cb)(void *, const char *), - void *cbopaque, unsigned i, bool bins, bool large); + void *cbopaque, unsigned i, bool bins, bool large, bool huge); /******************************************************************************/ @@ -58,62 +60,55 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i) { size_t page; - bool config_tcache; - unsigned nbins, j, gap_start; + bool config_tcache, in_gap; + unsigned nbins, j; CTL_GET("arenas.page", &page, size_t); CTL_GET("config.tcache", &config_tcache, bool); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" - " ndalloc nrequests nfills nflushes" - " newruns reruns curruns\n"); + "bins: size ind allocated nmalloc" + " ndalloc nrequests curregs regs pgs" + " nfills nflushes newruns reruns" + " curruns\n"); } else { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" - " ndalloc newruns reruns curruns\n"); + "bins: size ind allocated nmalloc" + " ndalloc nrequests curregs regs pgs" + " newruns reruns curruns\n"); } CTL_GET("arenas.nbins", &nbins, unsigned); - for (j = 0, gap_start = UINT_MAX; j < nbins; j++) { + for (j = 0, in_gap = false; j < nbins; j++) { uint64_t nruns; CTL_IJ_GET("stats.arenas.0.bins.0.nruns", &nruns, uint64_t); - if (nruns == 0) { - if (gap_start == UINT_MAX) - gap_start = j; - } else { - size_t reg_size, run_size, allocated; + if (nruns == 0) + in_gap = true; + else { + size_t reg_size, run_size, curregs; uint32_t nregs; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t reruns; size_t curruns; - if (gap_start != UINT_MAX) { - if (j > gap_start + 1) { - /* Gap of more than one size class. */ - malloc_cprintf(write_cb, cbopaque, - "[%u..%u]\n", gap_start, - j - 1); - } else { - /* Gap of one size class. */ - malloc_cprintf(write_cb, cbopaque, - "[%u]\n", gap_start); - } - gap_start = UINT_MAX; + if (in_gap) { + malloc_cprintf(write_cb, cbopaque, + " ---\n"); + in_gap = false; } CTL_J_GET("arenas.bin.0.size", ®_size, size_t); CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t); CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.bins.0.allocated", - &allocated, size_t); CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc", &nmalloc, uint64_t); CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc", &ndalloc, uint64_t); + CTL_IJ_GET("stats.arenas.0.bins.0.curregs", + &curregs, size_t); + CTL_IJ_GET("stats.arenas.0.bins.0.nrequests", + &nrequests, uint64_t); if (config_tcache) { - CTL_IJ_GET("stats.arenas.0.bins.0.nrequests", - &nrequests, uint64_t); CTL_IJ_GET("stats.arenas.0.bins.0.nfills", &nfills, uint64_t); CTL_IJ_GET("stats.arenas.0.bins.0.nflushes", @@ -125,33 +120,28 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, size_t); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "%13u %5zu %4u %3zu %12zu %12"PRIu64 - " %12"PRIu64" %12"PRIu64" %12"PRIu64 + "%20zu %3u %12zu %12"PRIu64" %12"PRIu64 + " %12"PRIu64" %12zu %4u %3zu %12"PRIu64 " %12"PRIu64" %12"PRIu64" %12"PRIu64 " %12zu\n", - j, reg_size, nregs, run_size / page, - allocated, nmalloc, ndalloc, nrequests, - nfills, nflushes, nruns, reruns, curruns); + reg_size, j, curregs * reg_size, nmalloc, + ndalloc, nrequests, curregs, nregs, run_size + / page, nfills, nflushes, nruns, reruns, + curruns); } else { malloc_cprintf(write_cb, cbopaque, - "%13u %5zu %4u %3zu %12zu %12"PRIu64 - " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu\n", - j, reg_size, nregs, run_size / page, - allocated, nmalloc, ndalloc, nruns, reruns, - curruns); + "%20zu %3u %12zu %12"PRIu64" %12"PRIu64 + " %12"PRIu64" %12zu %4u %3zu %12"PRIu64 + " %12"PRIu64" %12zu\n", + reg_size, j, curregs * reg_size, nmalloc, + ndalloc, nrequests, curregs, nregs, + run_size / page, nruns, reruns, curruns); } } } - if (gap_start != UINT_MAX) { - if (j > gap_start + 1) { - /* Gap of more than one size class. */ - malloc_cprintf(write_cb, cbopaque, "[%u..%u]\n", - gap_start, j - 1); - } else { - /* Gap of one size class. */ - malloc_cprintf(write_cb, cbopaque, "[%u]\n", gap_start); - } + if (in_gap) { + malloc_cprintf(write_cb, cbopaque, + " ---\n"); } } @@ -159,16 +149,15 @@ static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i) { - size_t page, nlruns, j; - ssize_t gap_start; - - CTL_GET("arenas.page", &page, size_t); + unsigned nbins, nlruns, j; + bool in_gap; malloc_cprintf(write_cb, cbopaque, - "large: size pages nmalloc ndalloc nrequests" - " curruns\n"); - CTL_GET("arenas.nlruns", &nlruns, size_t); - for (j = 0, gap_start = -1; j < nlruns; j++) { + "large: size ind allocated nmalloc ndalloc" + " nrequests curruns\n"); + CTL_GET("arenas.nbins", &nbins, unsigned); + CTL_GET("arenas.nlruns", &nlruns, unsigned); + for (j = 0, in_gap = false; j < nlruns; j++) { uint64_t nmalloc, ndalloc, nrequests; size_t run_size, curruns; @@ -178,32 +167,82 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, uint64_t); CTL_IJ_GET("stats.arenas.0.lruns.0.nrequests", &nrequests, uint64_t); - if (nrequests == 0) { - if (gap_start == -1) - gap_start = j; - } else { + if (nrequests == 0) + in_gap = true; + else { CTL_J_GET("arenas.lrun.0.size", &run_size, size_t); CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns, size_t); - if (gap_start != -1) { - malloc_cprintf(write_cb, cbopaque, "[%zu]\n", - j - gap_start); - gap_start = -1; + if (in_gap) { + malloc_cprintf(write_cb, cbopaque, + " ---\n"); + in_gap = false; } malloc_cprintf(write_cb, cbopaque, - "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64 + "%20zu %3u %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 " %12zu\n", - run_size, run_size / page, nmalloc, ndalloc, - nrequests, curruns); + run_size, nbins + j, curruns * run_size, nmalloc, + ndalloc, nrequests, curruns); } } - if (gap_start != -1) - malloc_cprintf(write_cb, cbopaque, "[%zu]\n", j - gap_start); + if (in_gap) { + malloc_cprintf(write_cb, cbopaque, + " ---\n"); + } +} + +static void +stats_arena_hchunks_print(void (*write_cb)(void *, const char *), + void *cbopaque, unsigned i) +{ + unsigned nbins, nlruns, nhchunks, j; + bool in_gap; + + malloc_cprintf(write_cb, cbopaque, + "huge: size ind allocated nmalloc ndalloc" + " nrequests curhchunks\n"); + CTL_GET("arenas.nbins", &nbins, unsigned); + CTL_GET("arenas.nlruns", &nlruns, unsigned); + CTL_GET("arenas.nhchunks", &nhchunks, unsigned); + for (j = 0, in_gap = false; j < nhchunks; j++) { + uint64_t nmalloc, ndalloc, nrequests; + size_t hchunk_size, curhchunks; + + CTL_IJ_GET("stats.arenas.0.hchunks.0.nmalloc", &nmalloc, + uint64_t); + CTL_IJ_GET("stats.arenas.0.hchunks.0.ndalloc", &ndalloc, + uint64_t); + CTL_IJ_GET("stats.arenas.0.hchunks.0.nrequests", &nrequests, + uint64_t); + if (nrequests == 0) + in_gap = true; + else { + CTL_J_GET("arenas.hchunk.0.size", &hchunk_size, + size_t); + CTL_IJ_GET("stats.arenas.0.hchunks.0.curhchunks", + &curhchunks, size_t); + if (in_gap) { + malloc_cprintf(write_cb, cbopaque, + " ---\n"); + in_gap = false; + } + malloc_cprintf(write_cb, cbopaque, + "%20zu %3u %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 + " %12zu\n", + hchunk_size, nbins + nlruns + j, + curhchunks * hchunk_size, nmalloc, ndalloc, + nrequests, curhchunks); + } + } + if (in_gap) { + malloc_cprintf(write_cb, cbopaque, + " ---\n"); + } } static void stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, - unsigned i, bool bins, bool large) + unsigned i, bool bins, bool large, bool huge) { unsigned nthreads; const char *dss; @@ -236,42 +275,51 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, nmadvise, nmadvise == 1 ? "" : "s", purged); malloc_cprintf(write_cb, cbopaque, - " allocated nmalloc ndalloc nrequests\n"); + " allocated nmalloc ndalloc" + " nrequests\n"); CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t); CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t); CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t); CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t); malloc_cprintf(write_cb, cbopaque, - "small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + "small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 + "\n", small_allocated, small_nmalloc, small_ndalloc, small_nrequests); CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t); CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t); CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t); CTL_I_GET("stats.arenas.0.large.nrequests", &large_nrequests, uint64_t); malloc_cprintf(write_cb, cbopaque, - "large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + "large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 + "\n", large_allocated, large_nmalloc, large_ndalloc, large_nrequests); CTL_I_GET("stats.arenas.0.huge.allocated", &huge_allocated, size_t); CTL_I_GET("stats.arenas.0.huge.nmalloc", &huge_nmalloc, uint64_t); CTL_I_GET("stats.arenas.0.huge.ndalloc", &huge_ndalloc, uint64_t); CTL_I_GET("stats.arenas.0.huge.nrequests", &huge_nrequests, uint64_t); malloc_cprintf(write_cb, cbopaque, - "huge: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + "huge: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 + "\n", huge_allocated, huge_nmalloc, huge_ndalloc, huge_nrequests); malloc_cprintf(write_cb, cbopaque, - "total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + "total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 + "\n", small_allocated + large_allocated + huge_allocated, small_nmalloc + large_nmalloc + huge_nmalloc, small_ndalloc + large_ndalloc + huge_ndalloc, small_nrequests + large_nrequests + huge_nrequests); - malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", pactive * page); + malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", + pactive * page); CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); - malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); + malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", + mapped); if (bins) stats_arena_bins_print(write_cb, cbopaque, i); if (large) stats_arena_lruns_print(write_cb, cbopaque, i); + if (huge) + stats_arena_hchunks_print(write_cb, cbopaque, i); } void @@ -286,6 +334,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, bool unmerged = true; bool bins = true; bool large = true; + bool huge = true; /* * Refresh stats, in case mallctl() was called by the application. @@ -328,6 +377,9 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, case 'l': large = false; break; + case 'h': + huge = false; + break; default:; } } @@ -515,7 +567,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\nMerged arenas stats:\n"); stats_arena_print(write_cb, cbopaque, - narenas, bins, large); + narenas, bins, large, huge); } } } @@ -541,7 +593,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cbopaque, "\narenas[%u]:\n", i); stats_arena_print(write_cb, - cbopaque, i, bins, large); + cbopaque, i, bins, large, + huge); } } } diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index a8f7aed6..028a9710 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -321,7 +321,8 @@ TEST_BEGIN(test_arenas_constants) TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM); TEST_ARENAS_CONSTANT(size_t, page, PAGE); TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS); - TEST_ARENAS_CONSTANT(size_t, nlruns, nlclasses); + TEST_ARENAS_CONSTANT(unsigned, nlruns, nlclasses); + TEST_ARENAS_CONSTANT(unsigned, nhchunks, nhclasses); #undef TEST_ARENAS_CONSTANT } @@ -363,6 +364,23 @@ TEST_BEGIN(test_arenas_lrun_constants) } TEST_END +TEST_BEGIN(test_arenas_hchunk_constants) +{ + +#define TEST_ARENAS_HCHUNK_CONSTANT(t, name, expected) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("arenas.hchunk.0."#name, &name, &sz, NULL, \ + 0), 0, "Unexpected mallctl() failure"); \ + assert_zu_eq(name, expected, "Incorrect "#name" size"); \ +} while (0) + + TEST_ARENAS_HCHUNK_CONSTANT(size_t, size, chunksize); + +#undef TEST_ARENAS_HCHUNK_CONSTANT +} +TEST_END + TEST_BEGIN(test_arenas_extend) { unsigned narenas_before, arena, narenas_after; @@ -420,6 +438,7 @@ main(void) test_arenas_constants, test_arenas_bin_constants, test_arenas_lrun_constants, + test_arenas_hchunk_constants, test_arenas_extend, test_stats_arenas)); } diff --git a/test/unit/stats.c b/test/unit/stats.c index 78c78cd5..fd92d542 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -97,7 +97,7 @@ TEST_END TEST_BEGIN(test_stats_arenas_summary) { unsigned arena; - void *little, *large; + void *little, *large, *huge; uint64_t epoch; size_t sz; int expected = config_stats ? 0 : ENOENT; @@ -112,6 +112,8 @@ TEST_BEGIN(test_stats_arenas_summary) assert_ptr_not_null(little, "Unexpected mallocx() failure"); large = mallocx(arena_maxclass, 0); assert_ptr_not_null(large, "Unexpected mallocx() failure"); + huge = mallocx(chunksize, 0); + assert_ptr_not_null(huge, "Unexpected mallocx() failure"); assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, "Unexpected mallctl() failure"); @@ -139,6 +141,7 @@ TEST_BEGIN(test_stats_arenas_summary) dallocx(little, 0); dallocx(large, 0); + dallocx(huge, 0); } TEST_END @@ -251,11 +254,51 @@ TEST_BEGIN(test_stats_arenas_large) } TEST_END +TEST_BEGIN(test_stats_arenas_huge) +{ + unsigned arena; + void *p; + size_t sz, allocated; + uint64_t epoch, nmalloc, ndalloc; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(chunksize, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.huge.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.huge.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.huge.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_zu_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_zu_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + } + + dallocx(p, 0); +} +TEST_END + TEST_BEGIN(test_stats_arenas_bins) { unsigned arena; void *p; - size_t sz, allocated, curruns; + size_t sz, curruns, curregs; uint64_t epoch, nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t nruns, nreruns; int expected = config_stats ? 0 : ENOENT; @@ -273,9 +316,6 @@ TEST_BEGIN(test_stats_arenas_bins) assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, "Unexpected mallctl() failure"); - sz = sizeof(size_t); - assert_d_eq(mallctl("stats.arenas.0.bins.0.allocated", &allocated, &sz, - NULL, 0), expected, "Unexpected mallctl() result"); sz = sizeof(uint64_t); assert_d_eq(mallctl("stats.arenas.0.bins.0.nmalloc", &nmalloc, &sz, NULL, 0), expected, "Unexpected mallctl() result"); @@ -283,6 +323,9 @@ TEST_BEGIN(test_stats_arenas_bins) NULL, 0), expected, "Unexpected mallctl() result"); assert_d_eq(mallctl("stats.arenas.0.bins.0.nrequests", &nrequests, &sz, NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.bins.0.curregs", &curregs, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); assert_d_eq(mallctl("stats.arenas.0.bins.0.nfills", &nfills, &sz, NULL, 0), config_tcache ? expected : ENOENT, @@ -300,14 +343,14 @@ TEST_BEGIN(test_stats_arenas_bins) NULL, 0), expected, "Unexpected mallctl() result"); if (config_stats) { - assert_zu_gt(allocated, 0, - "allocated should be greater than zero"); assert_u64_gt(nmalloc, 0, "nmalloc should be greater than zero"); assert_u64_ge(nmalloc, ndalloc, "nmalloc should be at least as large as ndalloc"); assert_u64_gt(nrequests, 0, "nrequests should be greater than zero"); + assert_zu_gt(curregs, 0, + "allocated should be greater than zero"); if (config_tcache) { assert_u64_gt(nfills, 0, "At least one fill should have occurred"); @@ -336,7 +379,7 @@ TEST_BEGIN(test_stats_arenas_lruns) assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), 0, "Unexpected mallctl() failure"); - p = mallocx(SMALL_MAXCLASS+1, 0); + p = mallocx(LARGE_MINCLASS, 0); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, @@ -368,6 +411,46 @@ TEST_BEGIN(test_stats_arenas_lruns) } TEST_END +TEST_BEGIN(test_stats_arenas_hchunks) +{ + unsigned arena; + void *p; + uint64_t epoch, nmalloc, ndalloc; + size_t curhchunks, sz; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(chunksize, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.hchunks.0.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.hchunks.0.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.hchunks.0.curhchunks", &curhchunks, + &sz, NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_u64_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_u64_gt(curhchunks, 0, + "At least one chunk should be currently allocated"); + } + + dallocx(p, 0); +} +TEST_END + int main(void) { @@ -379,6 +462,8 @@ main(void) test_stats_arenas_summary, test_stats_arenas_small, test_stats_arenas_large, + test_stats_arenas_huge, test_stats_arenas_bins, - test_stats_arenas_lruns)); + test_stats_arenas_lruns, + test_stats_arenas_hchunks)); } From 0cdabd2d489133e3cea8a00bdb9a986b24e57a66 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 14 Oct 2014 22:19:21 -0700 Subject: [PATCH 154/352] Update size class documentation. --- doc/jemalloc.xml.in | 110 +++++++++++++++++++++++++++++++++----------- 1 file changed, 84 insertions(+), 26 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 8111fc1d..fc01ad1b 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -501,13 +501,11 @@ for (i = 0; i < nbins; i++) { possible to find metadata for user objects very quickly. User objects are broken into three categories according to size: - small, large, and huge. Small objects are smaller than one page. Large - objects are smaller than the chunk size. Huge objects are a multiple of - the chunk size. Small and large objects are managed entirely by arenas; - huge objects are additionally aggregated in a single data structure that is - shared by all threads. Huge objects are typically used by applications - infrequently enough that this single data structure is not a scalability - issue. + small, large, and huge. Small and large objects are managed entirely by + arenas; huge objects are additionally aggregated in a single data structure + that is shared by all threads. Huge objects are typically used by + applications infrequently enough that this single data structure is not a + scalability issue. Each chunk that is managed by an arena tracks its contents as runs of contiguous pages (unused, backing a set of small objects, or backing one @@ -516,18 +514,18 @@ for (i = 0; i < nbins; i++) { allocations in constant time. Small objects are managed in groups by page runs. Each run maintains - a frontier and free list to track which regions are in use. Allocation - requests that are no more than half the quantum (8 or 16, depending on - architecture) are rounded up to the nearest power of two that is at least - sizeof(double). All other small - object size classes are multiples of the quantum, spaced such that internal - fragmentation is limited to approximately 25% for all but the smallest size - classes. Allocation requests that are larger than the maximum small size - class, but small enough to fit in an arena-managed chunk (see the opt.lg_chunk option), are - rounded up to the nearest run size. Allocation requests that are too large - to fit in an arena-managed chunk are rounded up to the nearest multiple of - the chunk size. + a bitmap to track which regions are in use. Allocation requests that are no + more than half the quantum (8 or 16, depending on architecture) are rounded + up to the nearest power of two that is at least sizeof(double). All other object size + classes are multiples of the quantum, spaced such that there are four size + classes for each doubling in size, which limits internal fragmentation to + approximately 20% for all but the smallest size classes. Small size classes + are smaller than four times the page size, large size classes are smaller + than the chunk size (see the opt.lg_chunk option), and + huge size classes extend from the chunk size up to one size class less than + the full address space size. Allocations are packed tightly together, which can be an issue for multi-threaded applications. If you need to assure that allocations do not @@ -554,13 +552,13 @@ for (i = 0; i < nbins; i++) { - Small + Small lg [8] 16 - [16, 32, 48, ..., 128] + [16, 32, 48, 64, 80, 96, 112, 128] 32 @@ -580,17 +578,77 @@ for (i = 0; i < nbins; i++) { 512 - [2560, 3072, 3584] + [2560, 3072, 3584, 4096] + + + 1 KiB + [5 KiB, 6 KiB, 7 KiB, 8 KiB] + + + 2 KiB + [10 KiB, 12 KiB, 14 KiB] + + + Large + 2 KiB + [16 KiB] - Large 4 KiB - [4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB] + [20 KiB, 24 KiB, 28 KiB, 32 KiB] + + + 8 KiB + [40 KiB, 48 KiB, 54 KiB, 64 KiB] + + + 16 KiB + [80 KiB, 96 KiB, 112 KiB, 128 KiB] + + + 32 KiB + [160 KiB, 192 KiB, 224 KiB, 256 KiB] + + + 64 KiB + [320 KiB, 384 KiB, 448 KiB, 512 KiB] + + + 128 KiB + [640 KiB, 768 KiB, 896 KiB, 1024 KiB] + + + 256 KiB + [1280 KiB, 1536 KiB, 1792 KiB, 2048 KiB] + + + 512 KiB + [2560 KiB, 3072 KiB, 3584 KiB] + + + Huge + 512 KiB + [4 MiB] + + + 1 MiB + [5 MiB, 6 MiB, 7 MiB, 8 MiB] + + + 2 MiB + [10 MiB, 12 MiB, 14 MiB, 16 MiB] - Huge 4 MiB - [4 MiB, 8 MiB, 12 MiB, ...] + [20 MiB, 24 MiB, 28 MiB, 32 MiB] + + + 8 MiB + [40 MiB, 48 MiB, 56 MiB, 64 MiB] + + + ... + ... From 9b41ac909facf4f09bb1b637b78ba647348e572e Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 14 Oct 2014 22:20:00 -0700 Subject: [PATCH 155/352] Fix huge allocation statistics. --- doc/jemalloc.xml.in | 5 +- include/jemalloc/internal/arena.h | 10 +- include/jemalloc/internal/private_symbols.txt | 3 + src/arena.c | 303 +++++++++++++----- src/huge.c | 95 ++---- 5 files changed, 254 insertions(+), 162 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index fc01ad1b..71b4cd19 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1719,9 +1719,8 @@ malloc_conf = "xmalloc:true";]]> Pointer to a counter that contains an approximate count of the current number of bytes in active pages. The estimate may be - high, but never low, because each arena rounds up to the nearest - multiple of the chunk size when computing its contribution to the - counter. Note that the epoch mallctl has no bearing on this counter. Furthermore, counter consistency is maintained via atomic operations, so it is necessary to use an atomic operation in diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index c31c8d7d..16c04d25 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -338,9 +338,15 @@ extern size_t arena_maxclass; /* Max size class for arenas. */ extern unsigned nlclasses; /* Number of large size classes. */ extern unsigned nhclasses; /* Number of huge size classes. */ -void *arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t usize, - size_t alignment, bool *zero); +void *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, + bool *zero); void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize); +void arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, + size_t oldsize, size_t usize); +void arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, + size_t oldsize, size_t usize); +bool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, + size_t oldsize, size_t usize, bool *zero); void arena_purge_all(arena_t *arena); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, uint64_t prof_accumbytes); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 66d48221..8eec874f 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -13,6 +13,9 @@ arena_choose arena_choose_hard arena_chunk_alloc_huge arena_chunk_dalloc_huge +arena_chunk_ralloc_huge_expand +arena_chunk_ralloc_huge_shrink +arena_chunk_ralloc_huge_similar arena_cleanup arena_dalloc arena_dalloc_bin diff --git a/src/arena.c b/src/arena.c index 74c36323..586e3c76 100644 --- a/src/arena.c +++ b/src/arena.c @@ -411,52 +411,6 @@ arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, return (chunk); } -void * -arena_chunk_alloc_huge(arena_t *arena, void *new_addr, size_t usize, - size_t alignment, bool *zero) -{ - void *ret; - chunk_alloc_t *chunk_alloc; - chunk_dalloc_t *chunk_dalloc; - - malloc_mutex_lock(&arena->lock); - chunk_alloc = arena->chunk_alloc; - chunk_dalloc = arena->chunk_dalloc; - if (config_stats) { - index_t index = size2index(usize) - nlclasses - NBINS; - - /* Optimistically update stats prior to unlocking. */ - arena->stats.allocated_huge += usize; - arena->stats.nmalloc_huge++; - arena->stats.hstats[index].nmalloc++; - arena->stats.hstats[index].curhchunks++; - arena->stats.mapped += usize; - } - arena->nactive += (usize >> LG_PAGE); - malloc_mutex_unlock(&arena->lock); - - ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, - new_addr, usize, alignment, zero); - if (config_stats) { - if (ret != NULL) - stats_cactive_add(usize); - else { - index_t index = size2index(usize) - nlclasses - NBINS; - - malloc_mutex_lock(&arena->lock); - /* Revert optimistic stats updates. */ - arena->stats.allocated_huge -= usize; - arena->stats.nmalloc_huge--; - arena->stats.hstats[index].nmalloc--; - arena->stats.hstats[index].curhchunks--; - arena->stats.mapped -= usize; - malloc_mutex_unlock(&arena->lock); - } - } - - return (ret); -} - static arena_chunk_t * arena_chunk_init_hard(arena_t *arena) { @@ -528,41 +482,6 @@ arena_chunk_alloc(arena_t *arena) return (chunk); } -static void -arena_chunk_dalloc_internal(arena_t *arena, arena_chunk_t *chunk) -{ - chunk_dalloc_t *chunk_dalloc; - - chunk_dalloc = arena->chunk_dalloc; - malloc_mutex_unlock(&arena->lock); - chunk_dalloc((void *)chunk, chunksize, arena->ind); - malloc_mutex_lock(&arena->lock); - if (config_stats) - arena->stats.mapped -= chunksize; -} - -void -arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) -{ - chunk_dalloc_t *chunk_dalloc; - - malloc_mutex_lock(&arena->lock); - chunk_dalloc = arena->chunk_dalloc; - if (config_stats) { - index_t index = size2index(usize) - nlclasses - NBINS; - - arena->stats.ndalloc_huge++; - arena->stats.allocated_huge -= usize; - arena->stats.hstats[index].ndalloc++; - arena->stats.hstats[index].curhchunks--; - arena->stats.mapped -= usize; - stats_cactive_sub(usize); - } - arena->nactive -= (usize >> LG_PAGE); - malloc_mutex_unlock(&arena->lock); - chunk_dalloc(chunk, usize, arena->ind); -} - static void arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) { @@ -584,17 +503,237 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; + chunk_dalloc_t *chunk_dalloc; arena->spare = chunk; if (arena_mapbits_dirty_get(spare, map_bias) != 0) { arena_dirty_remove(arena, spare, map_bias, chunk_npages-map_bias); } - arena_chunk_dalloc_internal(arena, spare); + chunk_dalloc = arena->chunk_dalloc; + malloc_mutex_unlock(&arena->lock); + chunk_dalloc((void *)spare, chunksize, arena->ind); + malloc_mutex_lock(&arena->lock); + if (config_stats) + arena->stats.mapped -= chunksize; } else arena->spare = chunk; } +static void +arena_huge_malloc_stats_update(arena_t *arena, size_t usize) +{ + index_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.nmalloc_huge++; + arena->stats.allocated_huge += usize; + arena->stats.hstats[index].nmalloc++; + arena->stats.hstats[index].curhchunks++; +} + +static void +arena_huge_malloc_stats_update_undo(arena_t *arena, size_t usize) +{ + index_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.nmalloc_huge--; + arena->stats.allocated_huge -= usize; + arena->stats.hstats[index].nmalloc--; + arena->stats.hstats[index].curhchunks--; +} + +static void +arena_huge_dalloc_stats_update(arena_t *arena, size_t usize) +{ + index_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.ndalloc_huge++; + arena->stats.allocated_huge -= usize; + arena->stats.hstats[index].ndalloc++; + arena->stats.hstats[index].curhchunks--; +} + +static void +arena_huge_dalloc_stats_update_undo(arena_t *arena, size_t usize) +{ + index_t index = size2index(usize) - nlclasses - NBINS; + + cassert(config_stats); + + arena->stats.ndalloc_huge--; + arena->stats.allocated_huge += usize; + arena->stats.hstats[index].ndalloc--; + arena->stats.hstats[index].curhchunks++; +} + +static void +arena_huge_ralloc_stats_update(arena_t *arena, size_t oldsize, size_t usize) +{ + + arena_huge_dalloc_stats_update(arena, oldsize); + arena_huge_malloc_stats_update(arena, usize); +} + +static void +arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize, + size_t usize) +{ + + arena_huge_dalloc_stats_update_undo(arena, oldsize); + arena_huge_malloc_stats_update_undo(arena, usize); +} + +void * +arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, + bool *zero) +{ + void *ret; + chunk_alloc_t *chunk_alloc; + chunk_dalloc_t *chunk_dalloc; + size_t csize = CHUNK_CEILING(usize); + + malloc_mutex_lock(&arena->lock); + chunk_alloc = arena->chunk_alloc; + chunk_dalloc = arena->chunk_dalloc; + if (config_stats) { + /* Optimistically update stats prior to unlocking. */ + arena_huge_malloc_stats_update(arena, usize); + arena->stats.mapped += usize; + } + arena->nactive += (usize >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); + + ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL, + csize, alignment, zero); + if (ret == NULL) { + /* Revert optimistic stats updates. */ + malloc_mutex_lock(&arena->lock); + if (config_stats) { + arena_huge_malloc_stats_update_undo(arena, usize); + arena->stats.mapped -= usize; + } + arena->nactive -= (usize >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); + return (NULL); + } + + if (config_stats) + stats_cactive_add(usize); + + return (ret); +} + +void +arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) +{ + chunk_dalloc_t *chunk_dalloc; + + malloc_mutex_lock(&arena->lock); + chunk_dalloc = arena->chunk_dalloc; + if (config_stats) { + arena_huge_dalloc_stats_update(arena, usize); + arena->stats.mapped -= usize; + stats_cactive_sub(usize); + } + arena->nactive -= (usize >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); + chunk_dalloc(chunk, CHUNK_CEILING(usize), arena->ind); +} + +void +arena_chunk_ralloc_huge_similar(arena_t *arena, void *chunk, size_t oldsize, + size_t usize) +{ + + assert(CHUNK_CEILING(oldsize) == CHUNK_CEILING(usize)); + assert(oldsize != usize); + + malloc_mutex_lock(&arena->lock); + if (config_stats) + arena_huge_ralloc_stats_update(arena, oldsize, usize); + if (oldsize < usize) { + size_t udiff = usize - oldsize; + arena->nactive += udiff >> LG_PAGE; + if (config_stats) + stats_cactive_add(udiff); + } else { + size_t udiff = oldsize - usize; + arena->nactive -= udiff >> LG_PAGE; + if (config_stats) + stats_cactive_sub(udiff); + } + malloc_mutex_unlock(&arena->lock); +} + +void +arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, + size_t usize) +{ + chunk_dalloc_t *chunk_dalloc; + size_t udiff = oldsize - usize; + size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); + + malloc_mutex_lock(&arena->lock); + chunk_dalloc = arena->chunk_dalloc; + if (config_stats) { + arena_huge_ralloc_stats_update(arena, oldsize, usize); + if (cdiff != 0) { + arena->stats.mapped -= cdiff; + stats_cactive_sub(udiff); + } + } + arena->nactive -= udiff >> LG_PAGE; + malloc_mutex_unlock(&arena->lock); + if (cdiff != 0) + chunk_dalloc(chunk + CHUNK_CEILING(usize), cdiff, arena->ind); +} + +bool +arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, + size_t usize, bool *zero) +{ + chunk_alloc_t *chunk_alloc; + chunk_dalloc_t *chunk_dalloc; + size_t udiff = usize - oldsize; + size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); + + malloc_mutex_lock(&arena->lock); + chunk_alloc = arena->chunk_alloc; + chunk_dalloc = arena->chunk_dalloc; + if (config_stats) { + /* Optimistically update stats prior to unlocking. */ + arena_huge_ralloc_stats_update(arena, oldsize, usize); + arena->stats.mapped += cdiff; + } + arena->nactive += (udiff >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); + + if (chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, chunk + + CHUNK_CEILING(oldsize), cdiff, chunksize, zero) == NULL) { + /* Revert optimistic stats updates. */ + malloc_mutex_lock(&arena->lock); + if (config_stats) { + arena_huge_ralloc_stats_update_undo(arena, + oldsize, usize); + arena->stats.mapped -= cdiff; + } + arena->nactive -= (udiff >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); + return (true); + } + + if (config_stats) + stats_cactive_add(udiff); + + return (false); +} + static arena_run_t * arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { diff --git a/src/huge.c b/src/huge.c index 5f46241d..740a93fc 100644 --- a/src/huge.c +++ b/src/huge.c @@ -31,15 +31,11 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, bool zero, bool try_tcache) { void *ret; - size_t csize; extent_node_t *node; bool is_zeroed; /* Allocate one or more contiguous chunks for this request. */ - csize = CHUNK_CEILING(usize); - assert(csize >= usize); - /* Allocate an extent node with which to track the chunk. */ node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), CACHELINE, false, try_tcache, NULL); @@ -56,7 +52,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, base_node_dalloc(node); return (NULL); } - ret = arena_chunk_alloc_huge(arena, NULL, csize, alignment, &is_zeroed); + ret = arena_chunk_alloc_huge(arena, usize, alignment, &is_zeroed); if (ret == NULL) { idalloct(tsd, node, try_tcache); return (NULL); @@ -104,25 +100,6 @@ huge_dalloc_junk(void *ptr, size_t usize) huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); #endif -static void -huge_ralloc_no_move_stats_update(arena_t *arena, size_t oldsize, size_t usize) -{ - index_t oldindex = size2index(oldsize) - nlclasses - NBINS; - index_t index = size2index(usize) - nlclasses - NBINS; - - cassert(config_stats); - - arena->stats.ndalloc_huge++; - arena->stats.allocated_huge -= oldsize; - arena->stats.hstats[oldindex].ndalloc++; - arena->stats.hstats[oldindex].curhchunks--; - - arena->stats.nmalloc_huge++; - arena->stats.allocated_huge += usize; - arena->stats.hstats[index].nmalloc++; - arena->stats.hstats[index].curhchunks++; -} - static void huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, size_t size, size_t extra, bool zero) @@ -135,34 +112,33 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize) usize = usize_next; - malloc_mutex_lock(&huge_mtx); + if (oldsize == usize) + return; + malloc_mutex_lock(&huge_mtx); key.addr = ptr; node = extent_tree_ad_search(&huge, &key); assert(node != NULL); assert(node->addr == ptr); - arena = node->arena; - - /* Update the size of the huge allocation if it changed. */ - if (oldsize != usize) { - assert(node->size != usize); - node->size = usize; - } - + /* Update the size of the huge allocation. */ + assert(node->size != usize); + node->size = usize; malloc_mutex_unlock(&huge_mtx); - /* Fill if necessary. */ + /* Fill if necessary (shrinking). */ + if (config_fill && unlikely(opt_junk) && oldsize > usize) + memset(ptr + usize, 0x5a, oldsize - usize); + + arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize); + + /* Fill if necessary (growing). */ if (oldsize < usize) { if (zero || (config_fill && unlikely(opt_zero))) memset(ptr + oldsize, 0, usize - oldsize); else if (config_fill && unlikely(opt_junk)) memset(ptr + oldsize, 0xa5, usize - oldsize); - } else if (config_fill && unlikely(opt_junk) && oldsize > usize) - memset(ptr + usize, 0x5a, oldsize - usize); - - if (config_stats) - huge_ralloc_no_move_stats_update(arena, oldsize, usize); + } } static void @@ -170,44 +146,28 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) { extent_node_t *node, key; arena_t *arena; - void *excess_addr; - size_t excess_size; malloc_mutex_lock(&huge_mtx); - key.addr = ptr; node = extent_tree_ad_search(&huge, &key); assert(node != NULL); assert(node->addr == ptr); - arena = node->arena; - /* Update the size of the huge allocation. */ node->size = usize; - malloc_mutex_unlock(&huge_mtx); - excess_addr = node->addr + CHUNK_CEILING(usize); - excess_size = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); - /* Zap the excess chunks. */ huge_dalloc_junk(ptr + usize, oldsize - usize); - if (excess_size > 0) - arena_chunk_dalloc_huge(arena, excess_addr, excess_size); - - if (config_stats) - huge_ralloc_no_move_stats_update(arena, oldsize, usize); + arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize); } static bool huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { size_t usize; - void *expand_addr; - size_t expand_size; extent_node_t *node, key; arena_t *arena; bool is_zeroed; - void *ret; usize = s2u(size); if (usize == 0) { @@ -215,19 +175,12 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { return (true); } - expand_addr = ptr + CHUNK_CEILING(oldsize); - expand_size = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); - assert(expand_size > 0); - malloc_mutex_lock(&huge_mtx); - key.addr = ptr; node = extent_tree_ad_search(&huge, &key); assert(node != NULL); assert(node->addr == ptr); - arena = node->arena; - malloc_mutex_unlock(&huge_mtx); /* @@ -235,12 +188,10 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { * it is possible to make correct junk/zero fill decisions below. */ is_zeroed = zero; - ret = arena_chunk_alloc_huge(arena, expand_addr, expand_size, chunksize, - &is_zeroed); - if (ret == NULL) - return (true); - assert(ret == expand_addr); + if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize, + &is_zeroed)) + return (true); malloc_mutex_lock(&huge_mtx); /* Update the size of the huge allocation. */ @@ -254,9 +205,6 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { memset(ptr + oldsize, 0, usize - oldsize); } - if (config_stats) - huge_ralloc_no_move_stats_update(arena, oldsize, usize); - return (false); } @@ -363,19 +311,16 @@ huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache) extent_node_t *node, key; malloc_mutex_lock(&huge_mtx); - /* Extract from tree of huge allocations. */ key.addr = ptr; node = extent_tree_ad_search(&huge, &key); assert(node != NULL); assert(node->addr == ptr); extent_tree_ad_remove(&huge, node); - malloc_mutex_unlock(&huge_mtx); huge_dalloc_junk(node->addr, node->size); - arena_chunk_dalloc_huge(node->arena, node->addr, - CHUNK_CEILING(node->size)); + arena_chunk_dalloc_huge(node->arena, node->addr, node->size); idalloct(tsd, node, try_tcache); } From d1f3ab4008f95e8928777d28a40aff9708701503 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 14 Oct 2014 22:31:49 -0700 Subject: [PATCH 156/352] Fix line wrapping. --- INSTALL | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/INSTALL b/INSTALL index a00960aa..b8459a81 100644 --- a/INSTALL +++ b/INSTALL @@ -191,8 +191,8 @@ any of the following arguments (not a definitive list) to 'configure': --with-lg-page= Specify the base 2 log of the system page size. This option is only useful - when cross compiling, since the configure script automatically determines the - host's page size by default. + when cross compiling, since the configure script automatically determines + the host's page size by default. --with-lg-page-sizes= Specify the comma-separated base 2 logs of the page sizes to support. This @@ -243,16 +243,16 @@ any of the following arguments (not a definitive list) to 'configure': safe values for the most commonly used modern architectures, there is a wrinkle related to GNU libc (glibc) that may impact your choice of . On most modern architectures, this mandates 16-byte alignment - (=4), but the glibc developers chose not to meet this requirement - for performance reasons. An old discussion can be found at + (=4), but the glibc developers chose not to meet this + requirement for performance reasons. An old discussion can be found at https://sourceware.org/bugzilla/show_bug.cgi?id=206 . Unlike glibc, jemalloc does follow the C standard by default (caveat: jemalloc - technically cheats if --with-lg-tiny-min is smaller than --with-lg-quantum), - but the fact that Linux systems already work around this allocator - noncompliance means that it is generally safe in practice to let jemalloc's - minimum alignment follow glibc's lead. If you specify --with-lg-quantum=3 - during configuration, jemalloc will provide additional size classes that - are not 16-byte-aligned (24, 40, and 56, assuming + technically cheats if --with-lg-tiny-min is smaller than + --with-lg-quantum), but the fact that Linux systems already work around + this allocator noncompliance means that it is generally safe in practice to + let jemalloc's minimum alignment follow glibc's lead. If you specify + --with-lg-quantum=3 during configuration, jemalloc will provide additional + size classes that are not 16-byte-aligned (24, 40, and 56, assuming --with-lg-size-class-group=2). --with-lg-tiny-min= From acbcbad1e18d3082ee6ce851994ed03f63ae55bd Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 15 Oct 2014 14:49:14 -0700 Subject: [PATCH 157/352] Thwart compiler optimizations. --- test/stress/microbench.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/stress/microbench.c b/test/stress/microbench.c index 980eca41..aefbe6a7 100644 --- a/test/stress/microbench.c +++ b/test/stress/microbench.c @@ -114,6 +114,10 @@ malloc_mus_free(void) void *p; p = malloc(1); + if (p == NULL) { + test_fail("Unexpected malloc() failure"); + return; + } malloc_usable_size(p); free(p); } @@ -124,6 +128,10 @@ malloc_sallocx_free(void) void *p; p = malloc(1); + if (p == NULL) { + test_fail("Unexpected malloc() failure"); + return; + } if (sallocx(p, 0) < 1) test_fail("Unexpected sallocx() failure"); free(p); @@ -143,6 +151,10 @@ malloc_nallocx_free(void) void *p; p = malloc(1); + if (p == NULL) { + test_fail("Unexpected malloc() failure"); + return; + } if (nallocx(1, 0) < 1) test_fail("Unexpected nallocx() failure"); free(p); From bf8d6a109200bf10f1c942ad914aa8cb5f279e17 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 15 Oct 2014 16:18:42 -0700 Subject: [PATCH 158/352] Add small run utilization to stats output. Add the 'util' column, which reports the proportion of available regions that are currently in use for each small size class. Small run utilization is the complement of external fragmentation. For example, utilization of 0.75 indicates that 25% of small run memory is consumed by external fragmentation, in other (more obtuse) words, 33% external fragmentation overhead. This resolves #27. --- src/stats.c | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/src/stats.c b/src/stats.c index 16a18c50..054f0332 100644 --- a/src/stats.c +++ b/src/stats.c @@ -69,14 +69,14 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, if (config_tcache) { malloc_cprintf(write_cb, cbopaque, "bins: size ind allocated nmalloc" - " ndalloc nrequests curregs regs pgs" - " nfills nflushes newruns reruns" - " curruns\n"); + " ndalloc nrequests curregs curruns regs" + " pgs util nfills nflushes newruns" + " reruns\n"); } else { malloc_cprintf(write_cb, cbopaque, "bins: size ind allocated nmalloc" - " ndalloc nrequests curregs regs pgs" - " newruns reruns curruns\n"); + " ndalloc nrequests curregs curruns regs" + " pgs util newruns reruns\n"); } CTL_GET("arenas.nbins", &nbins, unsigned); for (j = 0, in_gap = false; j < nbins; j++) { @@ -86,11 +86,12 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, if (nruns == 0) in_gap = true; else { - size_t reg_size, run_size, curregs; + size_t reg_size, run_size, curregs, availregs, milli; + size_t curruns; uint32_t nregs; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t reruns; - size_t curruns; + char util[6]; /* "x.yyy". */ if (in_gap) { malloc_cprintf(write_cb, cbopaque, @@ -118,24 +119,41 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, uint64_t); CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns, size_t); + + availregs = nregs * curruns; + milli = (availregs != 0) ? (1000 * curregs) / availregs + : 1000; + assert(milli <= 1000); + if (milli < 10) { + malloc_snprintf(util, sizeof(util), "0.00%zu", + milli); + } else if (milli < 100) { + malloc_snprintf(util, sizeof(util), "0.0%zu", + milli); + } else if (milli < 1000) { + malloc_snprintf(util, sizeof(util), "0.%zu", + milli); + } else + malloc_snprintf(util, sizeof(util), "1"); + if (config_tcache) { malloc_cprintf(write_cb, cbopaque, "%20zu %3u %12zu %12"PRIu64" %12"PRIu64 - " %12"PRIu64" %12zu %4u %3zu %12"PRIu64 + " %12"PRIu64" %12zu %12zu %4u %3zu %-5s" " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu\n", + " %12"PRIu64"\n", reg_size, j, curregs * reg_size, nmalloc, - ndalloc, nrequests, curregs, nregs, run_size - / page, nfills, nflushes, nruns, reruns, - curruns); + ndalloc, nrequests, curregs, curruns, nregs, + run_size / page, util, nfills, nflushes, + nruns, reruns); } else { malloc_cprintf(write_cb, cbopaque, "%20zu %3u %12zu %12"PRIu64" %12"PRIu64 - " %12"PRIu64" %12zu %4u %3zu %12"PRIu64 - " %12"PRIu64" %12zu\n", + " %12"PRIu64" %12zu %12zu %4u %3zu %-5s" + " %12"PRIu64" %12"PRIu64"\n", reg_size, j, curregs * reg_size, nmalloc, - ndalloc, nrequests, curregs, nregs, - run_size / page, nruns, reruns, curruns); + ndalloc, nrequests, curregs, curruns, nregs, + run_size / page, util, nruns, reruns); } } } From 9673983443a0782d975fbcb5d8457cfd411b8b56 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 15 Oct 2014 18:02:02 -0700 Subject: [PATCH 159/352] Purge/zero sub-chunk huge allocations as necessary. Purge trailing pages during shrinking huge reallocation when resulting size is not a multiple of the chunk size. Similarly, zero pages if necessary during growing huge reallocation when the resulting size is not a multiple of the chunk size. --- src/huge.c | 75 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 24 deletions(-) diff --git a/src/huge.c b/src/huge.c index 740a93fc..1734ff6e 100644 --- a/src/huge.c +++ b/src/huge.c @@ -61,18 +61,18 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, /* Insert node into huge. */ node->addr = ret; node->size = usize; + node->zeroed = is_zeroed; node->arena = arena; malloc_mutex_lock(&huge_mtx); extent_tree_ad_insert(&huge, node); malloc_mutex_unlock(&huge_mtx); - if (config_fill && !zero) { - if (unlikely(opt_junk)) - memset(ret, 0xa5, usize); - else if (unlikely(opt_zero) && !is_zeroed) + if (zero || (config_fill && unlikely(opt_zero))) { + if (!is_zeroed) memset(ret, 0, usize); - } + } else if (config_fill && unlikely(opt_junk)) + memset(ret, 0xa5, usize); return (ret); } @@ -105,6 +105,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, size_t size, size_t extra, bool zero) { size_t usize_next; + bool zeroed; extent_node_t *node, key; arena_t *arena; @@ -115,6 +116,17 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, if (oldsize == usize) return; + /* Fill if necessary (shrinking). */ + if (oldsize > usize) { + size_t sdiff = CHUNK_CEILING(usize) - usize; + zeroed = (sdiff != 0) ? !pages_purge(ptr + usize, sdiff) : true; + if (config_fill && unlikely(opt_junk)) { + memset(ptr + usize, 0x5a, oldsize - usize); + zeroed = false; + } + } else + zeroed = true; + malloc_mutex_lock(&huge_mtx); key.addr = ptr; node = extent_tree_ad_search(&huge, &key); @@ -124,19 +136,18 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, /* Update the size of the huge allocation. */ assert(node->size != usize); node->size = usize; + /* Clear node->zeroed if zeroing failed above. */ + node->zeroed = (node->zeroed && zeroed); malloc_mutex_unlock(&huge_mtx); - /* Fill if necessary (shrinking). */ - if (config_fill && unlikely(opt_junk) && oldsize > usize) - memset(ptr + usize, 0x5a, oldsize - usize); - arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize); /* Fill if necessary (growing). */ if (oldsize < usize) { - if (zero || (config_fill && unlikely(opt_zero))) - memset(ptr + oldsize, 0, usize - oldsize); - else if (config_fill && unlikely(opt_junk)) + if (zero || (config_fill && unlikely(opt_zero))) { + if (!zeroed) + memset(ptr + oldsize, 0, usize - oldsize); + } else if (config_fill && unlikely(opt_junk)) memset(ptr + oldsize, 0xa5, usize - oldsize); } } @@ -144,9 +155,18 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, static void huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) { + size_t sdiff; + bool zeroed; extent_node_t *node, key; arena_t *arena; + sdiff = CHUNK_CEILING(usize) - usize; + zeroed = (sdiff != 0) ? !pages_purge(ptr + usize, sdiff) : true; + if (config_fill && unlikely(opt_junk)) { + huge_dalloc_junk(ptr + usize, oldsize - usize); + zeroed = false; + } + malloc_mutex_lock(&huge_mtx); key.addr = ptr; node = extent_tree_ad_search(&huge, &key); @@ -155,10 +175,11 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) arena = node->arena; /* Update the size of the huge allocation. */ node->size = usize; + /* Clear node->zeroed if zeroing failed above. */ + node->zeroed = (node->zeroed && zeroed); malloc_mutex_unlock(&huge_mtx); /* Zap the excess chunks. */ - huge_dalloc_junk(ptr + usize, oldsize - usize); arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize); } @@ -167,7 +188,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { size_t usize; extent_node_t *node, key; arena_t *arena; - bool is_zeroed; + bool is_zeroed_subchunk, is_zeroed_chunk; usize = s2u(size); if (usize == 0) { @@ -181,16 +202,17 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { assert(node != NULL); assert(node->addr == ptr); arena = node->arena; + is_zeroed_subchunk = node->zeroed; malloc_mutex_unlock(&huge_mtx); /* - * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that - * it is possible to make correct junk/zero fill decisions below. + * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so + * that it is possible to make correct junk/zero fill decisions below. */ - is_zeroed = zero; + is_zeroed_chunk = zero; if (arena_chunk_ralloc_huge_expand(arena, ptr, oldsize, usize, - &is_zeroed)) + &is_zeroed_chunk)) return (true); malloc_mutex_lock(&huge_mtx); @@ -198,12 +220,17 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { node->size = usize; malloc_mutex_unlock(&huge_mtx); - if (config_fill && !zero) { - if (unlikely(opt_junk)) - memset(ptr + oldsize, 0xa5, usize - oldsize); - else if (unlikely(opt_zero) && !is_zeroed) - memset(ptr + oldsize, 0, usize - oldsize); - } + if (zero || (config_fill && unlikely(opt_zero))) { + if (!is_zeroed_subchunk) { + memset(ptr + oldsize, 0, CHUNK_CEILING(oldsize) - + oldsize); + } + if (!is_zeroed_chunk) { + memset(ptr + CHUNK_CEILING(oldsize), 0, usize - + CHUNK_CEILING(oldsize)); + } + } else if (config_fill && unlikely(opt_junk)) + memset(ptr + oldsize, 0xa5, usize - oldsize); return (false); } From c83bccd27396cbb6e818d83cc360a58aef96558d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 16 Oct 2014 12:33:18 -0700 Subject: [PATCH 160/352] Initialize chunks_mtx for all configurations. This resolves #150. --- src/chunk.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/chunk.c b/src/chunk.c index f65b67af..a7761162 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -409,11 +409,10 @@ chunk_boot(void) chunksize_mask = chunksize - 1; chunk_npages = (chunksize >> LG_PAGE); - if (config_stats || config_prof) { - if (malloc_mutex_init(&chunks_mtx)) - return (true); + if (malloc_mutex_init(&chunks_mtx)) + return (true); + if (config_stats || config_prof) memset(&stats_chunks, 0, sizeof(chunk_stats_t)); - } if (have_dss && chunk_dss_boot()) return (true); extent_tree_szad_new(&chunks_szad_mmap); From a9ea10d27c320926cab2e59c66ebcd25c49df24c Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Thu, 16 Oct 2014 15:05:02 -0400 Subject: [PATCH 161/352] use sized deallocation internally for ralloc The size of the source allocation is known at this point, so reading the chunk header can be avoided for the small size class fast path. This is not very useful right now, but it provides a significant performance boost with an alternate ralloc entry point taking the old size. --- src/arena.c | 2 +- src/huge.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index 586e3c76..d7377aec 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2220,7 +2220,7 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, copysize = (size < oldsize) ? size : oldsize; JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); - iqalloc(tsd, ptr, try_tcache_dalloc); + isqalloc(tsd, ptr, oldsize, try_tcache_dalloc); return (ret); } diff --git a/src/huge.c b/src/huge.c index 1734ff6e..826464c2 100644 --- a/src/huge.c +++ b/src/huge.c @@ -328,7 +328,7 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); - iqalloc(tsd, ptr, try_tcache_dalloc); + isqalloc(tsd, ptr, oldsize, try_tcache_dalloc); return (ret); } From 79725aa6f6823bf0703374cb4b89b64133321138 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Mon, 20 Oct 2014 14:08:37 -0200 Subject: [PATCH 162/352] Fix variable declaration with no type in the configure script. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index a7bf1039..5c51f27f 100644 --- a/configure.ac +++ b/configure.ac @@ -1363,7 +1363,7 @@ if test "x${enable_zone_allocator}" = "x1" ; then AC_DEFUN([JE_ZONE_PROGRAM], [AC_LANG_PROGRAM( [#include ], - [static foo[[sizeof($1) $2 sizeof(void *) * $3 ? 1 : -1]]] + [static int foo[[sizeof($1) $2 sizeof(void *) * $3 ? 1 : -1]]] )]) AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,14)],[JEMALLOC_ZONE_VERSION=3],[ From af1f5927633ee2cb98c095de0fcc67b8aacdc9c0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 30 Oct 2014 16:38:08 -0700 Subject: [PATCH 163/352] Use JEMALLOC_INLINE_C everywhere it's appropriate. --- src/arena.c | 16 ++++++++-------- src/ctl.c | 6 +++--- src/extent.c | 4 ++-- src/prof.c | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/arena.c b/src/arena.c index d7377aec..795f5302 100644 --- a/src/arena.c +++ b/src/arena.c @@ -39,7 +39,7 @@ arena_miscelm_to_bits(arena_chunk_map_misc_t *miscelm) return arena_mapbits_get(chunk, pageind); } -static inline int +JEMALLOC_INLINE_C int arena_run_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) { uintptr_t a_miscelm = (uintptr_t)a; @@ -55,7 +55,7 @@ arena_run_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_misc_t, rb_link, arena_run_comp) -static inline int +JEMALLOC_INLINE_C int arena_avail_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) { int ret; @@ -139,7 +139,7 @@ arena_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, arena->ndirty -= npages; } -static inline void * +JEMALLOC_INLINE_C void * arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) { void *ret; @@ -159,7 +159,7 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) return (ret); } -static inline void +JEMALLOC_INLINE_C void arena_run_reg_dalloc(arena_run_t *run, void *ptr) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); @@ -185,7 +185,7 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) run->nfree++; } -static inline void +JEMALLOC_INLINE_C void arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) { @@ -195,7 +195,7 @@ arena_run_zero(arena_chunk_t *chunk, size_t run_ind, size_t npages) (npages << LG_PAGE)); } -static inline void +JEMALLOC_INLINE_C void arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) { @@ -203,7 +203,7 @@ arena_run_page_mark_zeroed(arena_chunk_t *chunk, size_t run_ind) << LG_PAGE)), PAGE); } -static inline void +JEMALLOC_INLINE_C void arena_run_page_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) { size_t i; @@ -834,7 +834,7 @@ arena_run_alloc_small(arena_t *arena, size_t size, index_t binind) return (arena_run_alloc_small_helper(arena, size, binind)); } -static inline void +JEMALLOC_INLINE_C void arena_maybe_purge(arena_t *arena) { size_t threshold; diff --git a/src/ctl.c b/src/ctl.c index 72598b3d..b367c9f6 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -16,14 +16,14 @@ static ctl_stats_t ctl_stats; /******************************************************************************/ /* Helpers for named and indexed nodes. */ -static inline const ctl_named_node_t * +JEMALLOC_INLINE_C const ctl_named_node_t * ctl_named_node(const ctl_node_t *node) { return ((node->named) ? (const ctl_named_node_t *)node : NULL); } -static inline const ctl_named_node_t * +JEMALLOC_INLINE_C const ctl_named_node_t * ctl_named_children(const ctl_named_node_t *node, int index) { const ctl_named_node_t *children = ctl_named_node(node->children); @@ -31,7 +31,7 @@ ctl_named_children(const ctl_named_node_t *node, int index) return (children ? &children[index] : NULL); } -static inline const ctl_indexed_node_t * +JEMALLOC_INLINE_C const ctl_indexed_node_t * ctl_indexed_node(const ctl_node_t *node) { diff --git a/src/extent.c b/src/extent.c index 8c09b486..ca852016 100644 --- a/src/extent.c +++ b/src/extent.c @@ -3,7 +3,7 @@ /******************************************************************************/ -static inline int +JEMALLOC_INLINE_C int extent_szad_comp(extent_node_t *a, extent_node_t *b) { int ret; @@ -25,7 +25,7 @@ extent_szad_comp(extent_node_t *a, extent_node_t *b) rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad, extent_szad_comp) -static inline int +JEMALLOC_INLINE_C int extent_ad_comp(extent_node_t *a, extent_node_t *b) { uintptr_t a_addr = (uintptr_t)a->addr; diff --git a/src/prof.c b/src/prof.c index 40163271..36ee7584 100644 --- a/src/prof.c +++ b/src/prof.c @@ -244,7 +244,7 @@ bt_init(prof_bt_t *bt, void **vec) bt->len = 0; } -static inline void +JEMALLOC_INLINE_C void prof_enter(prof_tdata_t *tdata) { @@ -256,7 +256,7 @@ prof_enter(prof_tdata_t *tdata) malloc_mutex_lock(&bt2gctx_mtx); } -static inline void +JEMALLOC_INLINE_C void prof_leave(prof_tdata_t *tdata) { bool idump, gdump; From c93ed81cd06ae46906ae7a386fd6312caca391fb Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 30 Oct 2014 16:50:33 -0700 Subject: [PATCH 164/352] Fix prof_{enter,leave}() calls to pass tdata_self. --- src/prof.c | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/prof.c b/src/prof.c index 36ee7584..71b0994a 100644 --- a/src/prof.c +++ b/src/prof.c @@ -245,10 +245,11 @@ bt_init(prof_bt_t *bt, void **vec) } JEMALLOC_INLINE_C void -prof_enter(prof_tdata_t *tdata) +prof_enter(tsd_t *tsd, prof_tdata_t *tdata) { cassert(config_prof); + assert(tdata == prof_tdata_get(tsd, false)); assert(!tdata->enq); tdata->enq = true; @@ -257,11 +258,12 @@ prof_enter(prof_tdata_t *tdata) } JEMALLOC_INLINE_C void -prof_leave(prof_tdata_t *tdata) +prof_leave(tsd_t *tsd, prof_tdata_t *tdata) { bool idump, gdump; cassert(config_prof); + assert(tdata == prof_tdata_get(tsd, false)); malloc_mutex_unlock(&bt2gctx_mtx); @@ -542,7 +544,8 @@ prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) } static void -prof_gctx_try_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata) +prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, + prof_tdata_t *tdata) { cassert(config_prof); @@ -554,14 +557,14 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata) * avoid a race between the main body of prof_tctx_destroy() and entry * into this function. */ - prof_enter(tdata); + prof_enter(tsd, tdata_self); malloc_mutex_lock(gctx->lock); assert(gctx->nlimbo != 0); if (tctx_tree_empty(&gctx->tctxs) && gctx->nlimbo == 1) { /* Remove gctx from bt2gctx. */ if (ckh_remove(tsd, &bt2gctx, &gctx->bt, NULL, NULL)) not_reached(); - prof_leave(tdata); + prof_leave(tsd, tdata_self); /* Destroy gctx. */ malloc_mutex_unlock(gctx->lock); idalloc(tsd, gctx); @@ -572,7 +575,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_gctx_t *gctx, prof_tdata_t *tdata) */ gctx->nlimbo--; malloc_mutex_unlock(gctx->lock); - prof_leave(tdata); + prof_leave(tsd, tdata_self); } } @@ -655,8 +658,10 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) destroy_gctx = false; } malloc_mutex_unlock(gctx->lock); - if (destroy_gctx) - prof_gctx_try_destroy(tsd, gctx, tdata); + if (destroy_gctx) { + prof_gctx_try_destroy(tsd, prof_tdata_get(tsd, false), gctx, + tdata); + } if (destroy_tdata) prof_tdata_destroy(tsd, tdata, false); @@ -679,18 +684,18 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, } btkey; bool new_gctx; - prof_enter(tdata); + prof_enter(tsd, tdata); if (ckh_search(&bt2gctx, bt, &btkey.v, &gctx.v)) { /* bt has never been seen before. Insert it. */ gctx.p = prof_gctx_create(tsd, bt); if (gctx.v == NULL) { - prof_leave(tdata); + prof_leave(tsd, tdata); return (true); } btkey.p = &gctx.p->bt; if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { /* OOM. */ - prof_leave(tdata); + prof_leave(tsd, tdata); idalloc(tsd, gctx.v); return (true); } @@ -705,7 +710,7 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, malloc_mutex_unlock(gctx.p->lock); new_gctx = false; } - prof_leave(tdata); + prof_leave(tsd, tdata); *p_btkey = btkey.v; *p_gctx = gctx.p; @@ -751,7 +756,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) ret.v = imalloc(tsd, sizeof(prof_tctx_t)); if (ret.p == NULL) { if (new_gctx) - prof_gctx_try_destroy(tsd, gctx, tdata); + prof_gctx_try_destroy(tsd, tdata, gctx, tdata); return (NULL); } ret.p->tdata = tdata; @@ -765,7 +770,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) malloc_mutex_unlock(tdata->lock); if (error) { if (new_gctx) - prof_gctx_try_destroy(tsd, gctx, tdata); + prof_gctx_try_destroy(tsd, tdata, gctx, tdata); idalloc(tsd, ret.v); return (NULL); } @@ -872,9 +877,9 @@ prof_bt_count(void) if (tdata == NULL) return (0); - prof_enter(tdata); + malloc_mutex_lock(&bt2gctx_mtx); bt_count = ckh_count(&bt2gctx); - prof_leave(tdata); + malloc_mutex_unlock(&bt2gctx_mtx); return (bt_count); } @@ -1155,7 +1160,7 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) if (prof_gctx_should_destroy(gctx)) { gctx->nlimbo++; malloc_mutex_unlock(gctx->lock); - prof_gctx_try_destroy(tsd, gctx, tdata); + prof_gctx_try_destroy(tsd, tdata, gctx, tdata); } else malloc_mutex_unlock(gctx->lock); } @@ -1398,7 +1403,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) return (true); malloc_mutex_lock(&prof_dump_mtx); - prof_enter(tdata); + prof_enter(tsd, tdata); /* * Put gctx's in limbo and clear their counters in preparation for @@ -1421,7 +1426,7 @@ prof_dump(tsd_t *tsd, bool propagate_err, const char *filename, bool leakcheck) leak_ngctx = 0; gctx_tree_iter(&gctxs, NULL, prof_gctx_merge_iter, (void *)&leak_ngctx); - prof_leave(tdata); + prof_leave(tsd, tdata); /* Create dump file. */ if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) From 809b0ac3919da60c20ad59517ef560d0df639f3b Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Thu, 23 Oct 2014 10:30:52 -0400 Subject: [PATCH 165/352] mark huge allocations as unlikely This cleans up the fast path a bit more by moving away more code. --- .../jemalloc/internal/jemalloc_internal.h.in | 20 +++++++++---------- include/jemalloc/internal/prof.h | 4 ++-- src/arena.c | 4 ++-- src/jemalloc.c | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 294e2cc1..3ce5aba8 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -655,7 +655,7 @@ sa2u(size_t size, size_t alignment) } /* Try for a large size class. */ - if (size <= arena_maxclass && alignment < chunksize) { + if (likely(size <= arena_maxclass) && likely(alignment < chunksize)) { /* * We can't achieve subpage alignment, so round up alignment * to the minimum that can actually be supported. @@ -805,7 +805,7 @@ imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) assert(size != 0); - if (size <= arena_maxclass) + if (likely(size <= arena_maxclass)) return (arena_malloc(tsd, arena, size, false, try_tcache)); else return (huge_malloc(tsd, arena, size, false, try_tcache)); @@ -822,7 +822,7 @@ JEMALLOC_ALWAYS_INLINE void * icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) { - if (size <= arena_maxclass) + if (likely(size <= arena_maxclass)) return (arena_malloc(tsd, arena, size, true, try_tcache)); else return (huge_malloc(tsd, arena, size, true, try_tcache)); @@ -847,12 +847,12 @@ ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, if (usize <= SMALL_MAXCLASS && alignment < PAGE) ret = arena_malloc(tsd, arena, usize, zero, try_tcache); else { - if (usize <= arena_maxclass) { + if (likely(usize <= arena_maxclass)) { arena = arena_choose(tsd, arena); if (unlikely(arena == NULL)) return (NULL); ret = arena_palloc(arena, usize, alignment, zero); - } else if (alignment <= chunksize) + } else if (likely(alignment <= chunksize)) ret = huge_malloc(tsd, arena, usize, zero, try_tcache); else { ret = huge_palloc(tsd, arena, usize, alignment, zero, @@ -887,7 +887,7 @@ isalloc(const void *ptr, bool demote) assert(config_prof || !demote); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) + if (likely(chunk != ptr)) ret = arena_salloc(ptr, demote); else ret = huge_salloc(ptr); @@ -936,7 +936,7 @@ idalloct(tsd_t *tsd, void *ptr, bool try_tcache) assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) + if (likely(chunk != ptr)) arena_dalloc(tsd, chunk, ptr, try_tcache); else huge_dalloc(tsd, ptr, try_tcache); @@ -950,7 +950,7 @@ isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) + if (likely(chunk != ptr)) arena_sdalloc(tsd, chunk, ptr, size, try_tcache); else huge_dalloc(tsd, ptr, try_tcache); @@ -1038,7 +1038,7 @@ iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero, zero, try_tcache_alloc, try_tcache_dalloc, arena)); } - if (size <= arena_maxclass) { + if (likely(size <= arena_maxclass)) { return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, alignment, zero, try_tcache_alloc, try_tcache_dalloc)); } else { @@ -1069,7 +1069,7 @@ ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) return (true); } - if (size <= arena_maxclass) + if (likely(size <= arena_maxclass)) return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero)); else return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero)); diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 5103146b..e0d5f104 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -361,7 +361,7 @@ prof_tctx_get(const void *ptr) assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) { + if (likely(chunk != ptr)) { /* Region. */ ret = arena_prof_tctx_get(ptr); } else @@ -379,7 +379,7 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx) assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) { + if (likely(chunk != ptr)) { /* Region. */ arena_prof_tctx_set(ptr, tctx); } else diff --git a/src/arena.c b/src/arena.c index 795f5302..347d58e4 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2095,7 +2095,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, size_t usize; /* Make sure extra can't cause size_t overflow. */ - if (extra >= arena_maxclass) + if (unlikely(extra >= arena_maxclass)) return (true); usize = s2u(size + extra); @@ -2142,7 +2142,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, /* * Avoid moving the allocation if the size class can be left the same. */ - if (oldsize <= arena_maxclass) { + if (likely(oldsize <= arena_maxclass)) { if (oldsize <= SMALL_MAXCLASS) { assert(arena_bin_info[size2index(oldsize)].reg_size == oldsize); diff --git a/src/jemalloc.c b/src/jemalloc.c index 45439595..f130e999 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -264,7 +264,7 @@ a0alloc(size_t size, bool zero) if (size == 0) size = 1; - if (size <= arena_maxclass) + if (likely(size <= arena_maxclass)) ret = arena_malloc(NULL, a0get(), size, zero, false); else ret = huge_malloc(NULL, a0get(), size, zero, false); @@ -295,7 +295,7 @@ a0free(void *ptr) return; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) + if (likely(chunk != ptr)) arena_dalloc(NULL, chunk, ptr, false); else huge_dalloc(NULL, ptr, false); From d33f834591a2459f22da7a165c524340b5fc3a0c Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Fri, 24 Oct 2014 13:18:57 -0400 Subject: [PATCH 166/352] avoid redundant chunk header reads * use sized deallocation in iralloct_realign * iralloc and ixalloc always need the old size, so pass it in from the caller where it's often already calculated --- .../jemalloc/internal/jemalloc_internal.h.in | 33 ++++++------ src/jemalloc.c | 54 +++++++++---------- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 3ce5aba8..6f13093f 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -790,12 +790,13 @@ void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); -void *iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, - bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); -void *iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, - bool zero); -bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, - bool zero); +void *iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, + size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena); +void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, + size_t alignment, bool zero); +bool ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) @@ -1013,21 +1014,18 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - iqalloc(tsd, ptr, try_tcache_dalloc); + isqalloc(tsd, ptr, oldsize, try_tcache_dalloc); return (p); } JEMALLOC_ALWAYS_INLINE void * -iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero, - bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) +iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, + bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) { - size_t oldsize; assert(ptr != NULL); assert(size != 0); - oldsize = isalloc(ptr, config_prof); - if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { /* @@ -1048,21 +1046,22 @@ iralloct(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero, } JEMALLOC_ALWAYS_INLINE void * -iralloc(tsd_t *tsd, void *ptr, size_t size, size_t alignment, bool zero) +iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, + bool zero) { - return (iralloct(tsd, ptr, size, alignment, zero, true, true, NULL)); + return (iralloct(tsd, ptr, oldsize, size, alignment, zero, true, true, + NULL)); } JEMALLOC_ALWAYS_INLINE bool -ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) +ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, + bool zero) { - size_t oldsize; assert(ptr != NULL); assert(size != 0); - oldsize = isalloc(ptr, config_prof); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { /* Existing object alignment is inadequate. */ diff --git a/src/jemalloc.c b/src/jemalloc.c index f130e999..7d559ef4 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1529,19 +1529,20 @@ label_return: } static void * -irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t usize, prof_tctx_t *tctx) +irealloc_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize, + prof_tctx_t *tctx) { void *p; if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = iralloc(tsd, oldptr, LARGE_MINCLASS, 0, false); + p = iralloc(tsd, oldptr, old_usize, LARGE_MINCLASS, 0, false); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else - p = iralloc(tsd, oldptr, usize, 0, false); + p = iralloc(tsd, oldptr, old_usize, usize, 0, false); return (p); } @@ -1555,9 +1556,9 @@ irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) old_tctx = prof_tctx_get(oldptr); tctx = prof_alloc_prep(tsd, usize, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) - p = irealloc_prof_sample(tsd, oldptr, usize, tctx); + p = irealloc_prof_sample(tsd, oldptr, old_usize, usize, tctx); else - p = iralloc(tsd, oldptr, usize, 0, false); + p = iralloc(tsd, oldptr, old_usize, usize, 0, false); if (p == NULL) return (NULL); prof_realloc(tsd, p, usize, tctx, true, old_usize, old_tctx); @@ -1630,9 +1631,7 @@ je_realloc(void *ptr, size_t size) malloc_thread_init(); tsd = tsd_fetch(); - if ((config_prof && opt_prof) || config_stats || - (config_valgrind && unlikely(in_valgrind))) - old_usize = isalloc(ptr, config_prof); + old_usize = isalloc(ptr, config_prof); if (config_valgrind && unlikely(in_valgrind)) old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); @@ -1643,7 +1642,7 @@ je_realloc(void *ptr, size_t size) if (config_stats || (config_valgrind && unlikely(in_valgrind))) usize = s2u(size); - ret = iralloc(tsd, ptr, size, 0, false); + ret = iralloc(tsd, ptr, old_usize, size, 0, false); } } else { /* realloc(NULL, size) is equivalent to malloc(size). */ @@ -1922,22 +1921,22 @@ label_oom: } static void * -irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t size, size_t alignment, - size_t usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, - arena_t *arena, prof_tctx_t *tctx) +irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, + size_t alignment, size_t usize, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc, arena_t *arena, prof_tctx_t *tctx) { void *p; if (tctx == NULL) return (NULL); if (usize <= SMALL_MAXCLASS) { - p = iralloct(tsd, oldptr, LARGE_MINCLASS, alignment, zero, - try_tcache_alloc, try_tcache_dalloc, arena); + p = iralloct(tsd, oldptr, old_usize, LARGE_MINCLASS, alignment, + zero, try_tcache_alloc, try_tcache_dalloc, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else { - p = iralloct(tsd, oldptr, size, alignment, zero, + p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); } @@ -1955,10 +1954,11 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, old_tctx = prof_tctx_get(oldptr); tctx = prof_alloc_prep(tsd, *usize, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irallocx_prof_sample(tsd, oldptr, size, alignment, *usize, - zero, try_tcache_alloc, try_tcache_dalloc, arena, tctx); + p = irallocx_prof_sample(tsd, oldptr, old_usize, size, + alignment, *usize, zero, try_tcache_alloc, + try_tcache_dalloc, arena, tctx); } else { - p = iralloct(tsd, oldptr, size, alignment, zero, + p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, try_tcache_alloc, try_tcache_dalloc, arena); } if (unlikely(p == NULL)) { @@ -1988,7 +1988,7 @@ je_rallocx(void *ptr, size_t size, int flags) void *p; tsd_t *tsd; size_t usize; - UNUSED size_t old_usize JEMALLOC_CC_SILENCE_INIT(0); + size_t old_usize; UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t alignment = MALLOCX_ALIGN_GET(flags); bool zero = flags & MALLOCX_ZERO; @@ -2016,9 +2016,7 @@ je_rallocx(void *ptr, size_t size, int flags) arena = NULL; } - if ((config_prof && opt_prof) || config_stats || - ((config_valgrind && unlikely(in_valgrind)))) - old_usize = isalloc(ptr, config_prof); + old_usize = isalloc(ptr, config_prof); if (config_valgrind && unlikely(in_valgrind)) old_rzsize = u2rz(old_usize); @@ -2030,8 +2028,8 @@ je_rallocx(void *ptr, size_t size, int flags) if (unlikely(p == NULL)) goto label_oom; } else { - p = iralloct(tsd, ptr, size, alignment, zero, try_tcache_alloc, - try_tcache_dalloc, arena); + p = iralloct(tsd, ptr, old_usize, size, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); if (unlikely(p == NULL)) goto label_oom; if (config_stats || (config_valgrind && unlikely(in_valgrind))) @@ -2061,7 +2059,7 @@ ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, { size_t usize; - if (ixalloc(ptr, size, extra, alignment, zero)) + if (ixalloc(ptr, old_usize, size, extra, alignment, zero)) return (old_usize); usize = isalloc(ptr, config_prof); @@ -2080,9 +2078,9 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, /* Use minimum usize to determine whether promotion may happen. */ if (((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= SMALL_MAXCLASS) { - if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= - size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), - alignment, zero)) + if (ixalloc(ptr, old_usize, SMALL_MAXCLASS+1, + (SMALL_MAXCLASS+1 >= size+extra) ? 0 : size+extra - + (SMALL_MAXCLASS+1), alignment, zero)) return (old_usize); usize = isalloc(ptr, config_prof); if (max_usize < LARGE_MINCLASS) From cfc5706f6977a48f3b82d69cd68aa1cf8802fb8d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 30 Oct 2014 23:18:45 -0700 Subject: [PATCH 167/352] Miscellaneous cleanups. --- include/jemalloc/internal/prof.h | 10 ++++------ src/jemalloc.c | 6 +++--- src/prof.c | 4 +++- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index e0d5f104..e0818849 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -361,10 +361,9 @@ prof_tctx_get(const void *ptr) assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) { - /* Region. */ + if (likely(chunk != ptr)) ret = arena_prof_tctx_get(ptr); - } else + else ret = huge_prof_tctx_get(ptr); return (ret); @@ -379,10 +378,9 @@ prof_tctx_set(const void *ptr, prof_tctx_t *tctx) assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) { - /* Region. */ + if (likely(chunk != ptr)) arena_prof_tctx_set(ptr, tctx); - } else + else huge_prof_tctx_set(ptr, tctx); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 7d559ef4..23947f42 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1243,7 +1243,7 @@ imalloc_prof(tsd_t *tsd, size_t usize) p = imalloc_prof_sample(tsd, usize, tctx); else p = imalloc(tsd, usize); - if (p == NULL) { + if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); return (NULL); } @@ -1329,7 +1329,7 @@ imemalign_prof(tsd_t *tsd, size_t alignment, size_t usize) p = imemalign_prof_sample(tsd, alignment, usize, tctx); else p = ipalloc(tsd, usize, alignment, false); - if (p == NULL) { + if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); return (NULL); } @@ -1457,7 +1457,7 @@ icalloc_prof(tsd_t *tsd, size_t usize) p = icalloc_prof_sample(tsd, usize, tctx); else p = icalloc(tsd, usize); - if (p == NULL) { + if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); return (NULL); } diff --git a/src/prof.c b/src/prof.c index 71b0994a..4f5d4054 100644 --- a/src/prof.c +++ b/src/prof.c @@ -204,7 +204,9 @@ prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated) } void -prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) { +prof_malloc_sample_object(const void *ptr, size_t usize, prof_tctx_t *tctx) +{ + prof_tctx_set(ptr, tctx); malloc_mutex_lock(tctx->tdata->lock); From dc652131110abb480df608d17b20cf5bd4cfe2d4 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Thu, 30 Oct 2014 23:23:16 -0400 Subject: [PATCH 168/352] rm unused arena wrangling from xallocx It has no use for the arena_t since unlike rallocx it never makes a new memory allocation. It's just an unused parameter in ixalloc_helper. --- src/jemalloc.c | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 23947f42..8b2ab8d7 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2055,7 +2055,7 @@ label_oom: JEMALLOC_ALWAYS_INLINE_C size_t ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, - size_t alignment, bool zero, arena_t *arena) + size_t alignment, bool zero) { size_t usize; @@ -2068,8 +2068,7 @@ ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, static size_t ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, - size_t alignment, size_t max_usize, bool zero, arena_t *arena, - prof_tctx_t *tctx) + size_t alignment, size_t max_usize, bool zero, prof_tctx_t *tctx) { size_t usize; @@ -2087,7 +2086,7 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, arena_prof_promoted(ptr, usize); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, - zero, arena); + zero); } return (usize); @@ -2095,7 +2094,7 @@ ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, JEMALLOC_ALWAYS_INLINE_C size_t ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, - size_t extra, size_t alignment, bool zero, arena_t *arena) + size_t extra, size_t alignment, bool zero) { size_t max_usize, usize; prof_tctx_t *old_tctx, *tctx; @@ -2112,10 +2111,10 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, tctx = prof_alloc_prep(tsd, max_usize, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { usize = ixallocx_prof_sample(ptr, old_usize, size, extra, - alignment, zero, max_usize, arena, tctx); + alignment, zero, max_usize, tctx); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, - zero, arena); + zero); } if (unlikely(usize == old_usize)) { prof_alloc_rollback(tsd, tctx, false); @@ -2134,7 +2133,6 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t alignment = MALLOCX_ALIGN_GET(flags); bool zero = flags & MALLOCX_ZERO; - arena_t *arena; assert(ptr != NULL); assert(size != 0); @@ -2143,22 +2141,16 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) malloc_thread_init(); tsd = tsd_fetch(); - if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { - unsigned arena_ind = MALLOCX_ARENA_GET(flags); - arena = arena_get(tsd, arena_ind, true, true); - } else - arena = NULL; - old_usize = isalloc(ptr, config_prof); if (config_valgrind && unlikely(in_valgrind)) old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { usize = ixallocx_prof(tsd, ptr, old_usize, size, extra, - alignment, zero, arena); + alignment, zero); } else { usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, - zero, arena); + zero); } if (unlikely(usize == old_usize)) goto label_not_resized; From 6da2e9d4f6fdccf5108296c99b2b839a4f474bae Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 31 Oct 2014 17:08:13 -0700 Subject: [PATCH 169/352] Fix arena_sdalloc() to use promoted size. --- include/jemalloc/internal/arena.h | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 16c04d25..8782b191 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -1020,9 +1020,9 @@ arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) assert(((uintptr_t)ptr & PAGE_MASK) == 0); if (try_tcache && size <= tcache_maxclass && likely((tcache = - tcache_get(tsd, false)) != NULL)) { + tcache_get(tsd, false)) != NULL)) tcache_dalloc_large(tcache, ptr, size); - } else + else arena_dalloc_large(chunk->arena, chunk, ptr); } } @@ -1031,18 +1031,26 @@ JEMALLOC_ALWAYS_INLINE void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache) { + index_t binind; tcache_t *tcache; assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); + if (config_prof && opt_prof) { + /* Use promoted size, not request size. */ + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + binind = arena_mapbits_binind_get(chunk, pageind); + size = index2size(binind); + } else + binind = size2index(size); + if (likely(size <= SMALL_MAXCLASS)) { /* Small allocation. */ if (likely(try_tcache) && likely((tcache = tcache_get(tsd, - false)) != NULL)) { - index_t binind = size2index(size); + false)) != NULL)) tcache_dalloc_small(tcache, ptr, binind); - } else { + else { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; arena_dalloc_small(chunk->arena, chunk, ptr, pageind); @@ -1051,9 +1059,9 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, assert(((uintptr_t)ptr & PAGE_MASK) == 0); if (try_tcache && size <= tcache_maxclass && (tcache = - tcache_get(tsd, false)) != NULL) { + tcache_get(tsd, false)) != NULL) tcache_dalloc_large(tcache, ptr, size); - } else + else arena_dalloc_large(chunk->arena, chunk, ptr); } } From d7a9bab92db5dd3acc02e4f58e95637c6338c285 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 31 Oct 2014 22:26:24 -0700 Subject: [PATCH 170/352] Fix arena_sdalloc() to use promoted size (second attempt). Unlike the preceeding attempted fix, this version avoids the potential for converting an invalid bin index to a size class. --- include/jemalloc/internal/arena.h | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 8782b191..a42522d6 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -1031,26 +1031,29 @@ JEMALLOC_ALWAYS_INLINE void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, bool try_tcache) { - index_t binind; tcache_t *tcache; assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); if (config_prof && opt_prof) { - /* Use promoted size, not request size. */ size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - binind = arena_mapbits_binind_get(chunk, pageind); - size = index2size(binind); - } else - binind = size2index(size); + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + if (arena_mapbits_large_get(chunk, pageind) != 0) { + /* Make sure to use promoted size, not request size. */ + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + size = arena_mapbits_large_size_get(chunk, pageind); + } + } + assert(s2u(size) == s2u(arena_salloc(ptr, false))); if (likely(size <= SMALL_MAXCLASS)) { /* Small allocation. */ if (likely(try_tcache) && likely((tcache = tcache_get(tsd, - false)) != NULL)) + false)) != NULL)) { + index_t binind = size2index(size); tcache_dalloc_small(tcache, ptr, binind); - else { + } else { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; arena_dalloc_small(chunk->arena, chunk, ptr, pageind); From 82cb603ed799f29e387f37fb44cdfbe98fd2e4ee Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 1 Nov 2014 00:20:28 -0700 Subject: [PATCH 171/352] Don't dereference NULL tdata in prof_{enter,leave}(). It is possible for the thread's tdata to be NULL late during thread destruction, so take care not to dereference a NULL pointer in such cases. --- src/prof.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/prof.c b/src/prof.c index 4f5d4054..1103cc94 100644 --- a/src/prof.c +++ b/src/prof.c @@ -253,8 +253,10 @@ prof_enter(tsd_t *tsd, prof_tdata_t *tdata) cassert(config_prof); assert(tdata == prof_tdata_get(tsd, false)); - assert(!tdata->enq); - tdata->enq = true; + if (tdata != NULL) { + assert(!tdata->enq); + tdata->enq = true; + } malloc_mutex_lock(&bt2gctx_mtx); } @@ -262,24 +264,27 @@ prof_enter(tsd_t *tsd, prof_tdata_t *tdata) JEMALLOC_INLINE_C void prof_leave(tsd_t *tsd, prof_tdata_t *tdata) { - bool idump, gdump; cassert(config_prof); assert(tdata == prof_tdata_get(tsd, false)); malloc_mutex_unlock(&bt2gctx_mtx); - assert(tdata->enq); - tdata->enq = false; - idump = tdata->enq_idump; - tdata->enq_idump = false; - gdump = tdata->enq_gdump; - tdata->enq_gdump = false; + if (tdata != NULL) { + bool idump, gdump; - if (idump) - prof_idump(); - if (gdump) - prof_gdump(); + assert(tdata->enq); + tdata->enq = false; + idump = tdata->enq_idump; + tdata->enq_idump = false; + gdump = tdata->enq_gdump; + tdata->enq_gdump = false; + + if (idump) + prof_idump(); + if (gdump) + prof_gdump(); + } } #ifdef JEMALLOC_PROF_LIBUNWIND From 2b2f6dc1e45808c31fb2f3ae33306d224ec0b2d2 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 1 Nov 2014 02:29:10 -0700 Subject: [PATCH 172/352] Disable arena_dirty_count() validation. --- src/arena.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index 347d58e4..ef42771a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -872,7 +872,7 @@ arena_dirty_count(arena_t *arena) ndirty += npages; } - return (ndirty); + return (ndirty); } static size_t @@ -1015,7 +1015,11 @@ arena_purge(arena_t *arena, bool all) size_t npurge, npurgeable, npurged; arena_chunk_miscelms_t purge_list; - if (config_debug) { + /* + * Calls to arena_dirty_count() are disabled even for debug builds + * because overhead grows nonlinearly as memory usage increases. + */ + if (false && config_debug) { size_t ndirty = arena_dirty_count(arena); assert(ndirty == arena->ndirty); } From c002a5c80058ee27acb234ef34f69b0cf6836836 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 4 Nov 2014 18:03:11 -0800 Subject: [PATCH 173/352] Fix two quarantine regressions. Fix quarantine to actually update tsd when expanding, and to avoid double initialization (leaking the first quarantine) due to recursive initialization. This resolves #161. --- include/jemalloc/internal/private_symbols.txt | 1 + include/jemalloc/internal/quarantine.h | 5 +++-- src/quarantine.c | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 8eec874f..1988c6ed 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -339,6 +339,7 @@ prof_thread_name_set quarantine quarantine_alloc_hook quarantine_cleanup +quarantine_alloc_hook_work quarantine_init register_zone rtree_delete diff --git a/include/jemalloc/internal/quarantine.h b/include/jemalloc/internal/quarantine.h index 4e9c710a..a399faaa 100644 --- a/include/jemalloc/internal/quarantine.h +++ b/include/jemalloc/internal/quarantine.h @@ -30,6 +30,7 @@ struct quarantine_s { #ifdef JEMALLOC_H_EXTERNS quarantine_t *quarantine_init(tsd_t *tsd, size_t lg_maxobjs); +void quarantine_alloc_hook_work(tsd_t *tsd); void quarantine(tsd_t *tsd, void *ptr); void quarantine_cleanup(tsd_t *tsd); @@ -50,8 +51,8 @@ quarantine_alloc_hook(void) assert(config_fill && opt_quarantine); tsd = tsd_fetch(); - if (tsd_quarantine_get(tsd) == NULL && tsd_nominal(tsd)) - tsd_quarantine_set(tsd, quarantine_init(tsd, LG_MAXOBJS_INIT)); + if (tsd_quarantine_get(tsd) == NULL) + quarantine_alloc_hook_work(tsd); } #endif diff --git a/src/quarantine.c b/src/quarantine.c index 1301b479..aa1c3b04 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -24,6 +24,8 @@ quarantine_init(tsd_t *tsd, size_t lg_maxobjs) { quarantine_t *quarantine; + assert(tsd_nominal(tsd)); + quarantine = (quarantine_t *)imalloc(tsd, offsetof(quarantine_t, objs) + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); if (quarantine == NULL) @@ -36,6 +38,25 @@ quarantine_init(tsd_t *tsd, size_t lg_maxobjs) return (quarantine); } +void +quarantine_alloc_hook_work(tsd_t *tsd) +{ + quarantine_t *quarantine; + + if (!tsd_nominal(tsd)) + return; + + quarantine = quarantine_init(tsd, LG_MAXOBJS_INIT); + /* + * Check again whether quarantine has been initialized, because + * qurantine_init() may have triggered recursive initialization. + */ + if (tsd_quarantine_get(tsd) == NULL) + tsd_quarantine_set(tsd, quarantine); + else + idalloc(tsd, quarantine); +} + static quarantine_t * quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) { @@ -67,6 +88,7 @@ quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) } idalloc(tsd, quarantine); + tsd_quarantine_set(tsd, ret); return (ret); } From 9cf2be0a81b77d4586591c19fb469a51fe6684fa Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 7 Nov 2014 14:50:38 -0800 Subject: [PATCH 174/352] Make quarantine_init() static. --- include/jemalloc/internal/private_symbols.txt | 3 +-- include/jemalloc/internal/quarantine.h | 1 - src/quarantine.c | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 1988c6ed..ee973c9f 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -338,9 +338,8 @@ prof_thread_name_get prof_thread_name_set quarantine quarantine_alloc_hook -quarantine_cleanup quarantine_alloc_hook_work -quarantine_init +quarantine_cleanup register_zone rtree_delete rtree_get diff --git a/include/jemalloc/internal/quarantine.h b/include/jemalloc/internal/quarantine.h index a399faaa..ae607399 100644 --- a/include/jemalloc/internal/quarantine.h +++ b/include/jemalloc/internal/quarantine.h @@ -29,7 +29,6 @@ struct quarantine_s { /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -quarantine_t *quarantine_init(tsd_t *tsd, size_t lg_maxobjs); void quarantine_alloc_hook_work(tsd_t *tsd); void quarantine(tsd_t *tsd, void *ptr); void quarantine_cleanup(tsd_t *tsd); diff --git a/src/quarantine.c b/src/quarantine.c index aa1c3b04..ddacc6ee 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -19,7 +19,7 @@ static void quarantine_drain(tsd_t *tsd, quarantine_t *quarantine, /******************************************************************************/ -quarantine_t * +static quarantine_t * quarantine_init(tsd_t *tsd, size_t lg_maxobjs) { quarantine_t *quarantine; From 2012d5a5601c787ce464fac0cbd2b16e3754cfa2 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 17 Nov 2014 09:54:49 -0800 Subject: [PATCH 175/352] Fix pointer arithmetic undefined behavior. Reported by Denis Denisov. --- src/arena.c | 11 +++++++---- src/huge.c | 37 ++++++++++++++++++++++++------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/src/arena.c b/src/arena.c index ef42771a..1ecc5d0b 100644 --- a/src/arena.c +++ b/src/arena.c @@ -690,8 +690,10 @@ arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, } arena->nactive -= udiff >> LG_PAGE; malloc_mutex_unlock(&arena->lock); - if (cdiff != 0) - chunk_dalloc(chunk + CHUNK_CEILING(usize), cdiff, arena->ind); + if (cdiff != 0) { + chunk_dalloc((void *)((uintptr_t)chunk + CHUNK_CEILING(usize)), + cdiff, arena->ind); + } } bool @@ -714,8 +716,9 @@ arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, arena->nactive += (udiff >> LG_PAGE); malloc_mutex_unlock(&arena->lock); - if (chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, chunk + - CHUNK_CEILING(oldsize), cdiff, chunksize, zero) == NULL) { + if (chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, + (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)), cdiff, + chunksize, zero) == NULL) { /* Revert optimistic stats updates. */ malloc_mutex_lock(&arena->lock); if (config_stats) { diff --git a/src/huge.c b/src/huge.c index 826464c2..7ad9b662 100644 --- a/src/huge.c +++ b/src/huge.c @@ -119,9 +119,11 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, /* Fill if necessary (shrinking). */ if (oldsize > usize) { size_t sdiff = CHUNK_CEILING(usize) - usize; - zeroed = (sdiff != 0) ? !pages_purge(ptr + usize, sdiff) : true; + zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + + usize), sdiff) : true; if (config_fill && unlikely(opt_junk)) { - memset(ptr + usize, 0x5a, oldsize - usize); + memset((void *)((uintptr_t)ptr + usize), 0x5a, oldsize - + usize); zeroed = false; } } else @@ -145,10 +147,14 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, /* Fill if necessary (growing). */ if (oldsize < usize) { if (zero || (config_fill && unlikely(opt_zero))) { - if (!zeroed) - memset(ptr + oldsize, 0, usize - oldsize); - } else if (config_fill && unlikely(opt_junk)) - memset(ptr + oldsize, 0xa5, usize - oldsize); + if (!zeroed) { + memset((void *)((uintptr_t)ptr + oldsize), 0, + usize - oldsize); + } + } else if (config_fill && unlikely(opt_junk)) { + memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - + oldsize); + } } } @@ -161,9 +167,11 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) arena_t *arena; sdiff = CHUNK_CEILING(usize) - usize; - zeroed = (sdiff != 0) ? !pages_purge(ptr + usize, sdiff) : true; + zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + usize), + sdiff) : true; if (config_fill && unlikely(opt_junk)) { - huge_dalloc_junk(ptr + usize, oldsize - usize); + huge_dalloc_junk((void *)((uintptr_t)ptr + usize), oldsize - + usize); zeroed = false; } @@ -222,15 +230,18 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { if (zero || (config_fill && unlikely(opt_zero))) { if (!is_zeroed_subchunk) { - memset(ptr + oldsize, 0, CHUNK_CEILING(oldsize) - - oldsize); + memset((void *)((uintptr_t)ptr + oldsize), 0, + CHUNK_CEILING(oldsize) - oldsize); } if (!is_zeroed_chunk) { - memset(ptr + CHUNK_CEILING(oldsize), 0, usize - + memset((void *)((uintptr_t)ptr + + CHUNK_CEILING(oldsize)), 0, usize - CHUNK_CEILING(oldsize)); } - } else if (config_fill && unlikely(opt_junk)) - memset(ptr + oldsize, 0xa5, usize - oldsize); + } else if (config_fill && unlikely(opt_junk)) { + memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - + oldsize); + } return (false); } From d49cb68b9e8b57169240e16686f4f60d6b5a089f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 17 Nov 2014 10:31:59 -0800 Subject: [PATCH 176/352] Fix more pointer arithmetic undefined behavior. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported by Guilherme Gonçalves. This resolves #166. --- src/arena.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/arena.c b/src/arena.c index 1ecc5d0b..f351c090 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2341,12 +2341,12 @@ arena_new(unsigned ind) if (config_stats) { memset(&arena->stats, 0, sizeof(arena_stats_t)); - arena->stats.lstats = (malloc_large_stats_t *)(((void *)arena) + - CACHELINE_CEILING(sizeof(arena_t))); + arena->stats.lstats = (malloc_large_stats_t *)((uintptr_t)arena + + CACHELINE_CEILING(sizeof(arena_t))); memset(arena->stats.lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); - arena->stats.hstats = (malloc_huge_stats_t *)(((void *)arena) + - CACHELINE_CEILING(sizeof(arena_t)) + + arena->stats.hstats = (malloc_huge_stats_t *)((uintptr_t)arena + + CACHELINE_CEILING(sizeof(arena_t)) + QUANTUM_CEILING(nlclasses * sizeof(malloc_large_stats_t))); memset(arena->stats.hstats, 0, nhclasses * sizeof(malloc_huge_stats_t)); From a2136025c4c4861b91f361a90c1dc94214848708 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Tue, 18 Nov 2014 18:48:48 -0200 Subject: [PATCH 177/352] Remove extra definition of je_tsd_boot on win32. --- include/jemalloc/internal/tsd.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index b5658f8e..35dd8628 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -364,12 +364,6 @@ a_name##tsd_boot(void) \ a_name##tsd_boot1(); \ return (false); \ } \ -a_attr bool \ -a_name##tsd_boot(void) \ -{ \ - \ - return (false); \ -} \ /* Get/set. */ \ a_attr a_type * \ a_name##tsd_get(void) \ From 879e76a9e57e725e927e77900940967d301a4958 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Mon, 3 Nov 2014 14:02:52 -0500 Subject: [PATCH 178/352] teach the dss chunk allocator to handle new_addr This provides in-place expansion of huge allocations when the end of the allocation is at the end of the sbrk heap. There's already the ability to extend in-place via recycled chunks but this handles the initial growth of the heap via repeated vector / string reallocations. A possible future extension could allow realloc to go from the following: | huge allocation | recycled chunks | ^ dss_end To a larger allocation built from recycled *and* new chunks: | huge allocation | ^ dss_end Doing that would involve teaching the chunk recycling code to request new chunks to satisfy the request. The chunk_dss code wouldn't require any further changes. #include int main(void) { size_t chunk = 4 * 1024 * 1024; void *ptr = NULL; for (size_t size = chunk; size < chunk * 128; size *= 2) { ptr = realloc(ptr, size); if (!ptr) return 1; } } dss:secondary: 0.083s dss:primary: 0.083s After: dss:secondary: 0.083s dss:primary: 0.003s The dss heap grows in the upwards direction, so the oldest chunks are at the low addresses and they are used first. Linux prefers to grow the mmap heap downwards, so the trick will not work in the *current* mmap chunk allocator as a huge allocation will only be at the top of the heap in a contrived case. --- include/jemalloc/internal/chunk_dss.h | 3 ++- src/chunk.c | 12 +++++------- src/chunk_dss.c | 11 ++++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/include/jemalloc/internal/chunk_dss.h b/include/jemalloc/internal/chunk_dss.h index 4535ce09..09896470 100644 --- a/include/jemalloc/internal/chunk_dss.h +++ b/include/jemalloc/internal/chunk_dss.h @@ -23,7 +23,8 @@ extern const char *dss_prec_names[]; dss_prec_t chunk_dss_prec_get(void); bool chunk_dss_prec_set(dss_prec_t dss_prec); -void *chunk_alloc_dss(size_t size, size_t alignment, bool *zero); +void *chunk_alloc_dss(void *new_addr, size_t size, size_t alignment, + bool *zero); bool chunk_in_dss(void *chunk); bool chunk_dss_boot(void); void chunk_dss_prefork(void); diff --git a/src/chunk.c b/src/chunk.c index a7761162..b3737180 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -154,16 +154,15 @@ chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, new_addr, size, alignment, base, zero)) != NULL) return (ret); - /* requesting an address only implemented for recycle */ - if (new_addr == NULL - && (ret = chunk_alloc_dss(size, alignment, zero)) != NULL) + if ((ret = chunk_alloc_dss(new_addr, size, alignment, zero)) + != NULL) return (ret); } /* mmap. */ if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, new_addr, size, alignment, base, zero)) != NULL) return (ret); - /* requesting an address only implemented for recycle */ + /* requesting an address not implemented for chunk_alloc_mmap */ if (new_addr == NULL && (ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) return (ret); @@ -172,9 +171,8 @@ chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, new_addr, size, alignment, base, zero)) != NULL) return (ret); - /* requesting an address only implemented for recycle */ - if (new_addr == NULL && - (ret = chunk_alloc_dss(size, alignment, zero)) != NULL) + if ((ret = chunk_alloc_dss(new_addr, size, alignment, zero)) + != NULL) return (ret); } diff --git a/src/chunk_dss.c b/src/chunk_dss.c index cce71041..edba3b23 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -66,7 +66,7 @@ chunk_dss_prec_set(dss_prec_t dss_prec) } void * -chunk_alloc_dss(size_t size, size_t alignment, bool *zero) +chunk_alloc_dss(void *new_addr, size_t size, size_t alignment, bool *zero) { void *ret; @@ -93,8 +93,17 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero) * malloc. */ do { + /* Avoid an unnecessary system call. */ + if (new_addr != NULL && dss_max != new_addr) + break; + /* Get the current end of the DSS. */ dss_max = chunk_dss_sbrk(0); + + /* Make sure the earlier condition still holds. */ + if (new_addr != NULL && dss_max != new_addr) + break; + /* * Calculate how much padding is necessary to * chunk-align the end of the DSS. From f79e01f75b79058c3be0ce6de0d46f8a9a990176 Mon Sep 17 00:00:00 2001 From: Yuriy Kaminskiy Date: Tue, 2 Dec 2014 16:24:11 -0800 Subject: [PATCH 179/352] Fix test_stats_arenas_bins for 32-bit builds. --- test/unit/stats.c | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/stats.c b/test/unit/stats.c index fd92d542..946e7370 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -327,6 +327,7 @@ TEST_BEGIN(test_stats_arenas_bins) assert_d_eq(mallctl("stats.arenas.0.bins.0.curregs", &curregs, &sz, NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); assert_d_eq(mallctl("stats.arenas.0.bins.0.nfills", &nfills, &sz, NULL, 0), config_tcache ? expected : ENOENT, "Unexpected mallctl() result"); From 1036ddbf11b7e9ec566b92b3dd50e105fc5f6932 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 4 Dec 2014 16:42:42 -0800 Subject: [PATCH 180/352] Fix OOM cleanup in huge_palloc(). Fix OOM cleanup in huge_palloc() to call idalloct() rather than base_node_dalloc(). This bug is a result of incomplete refactoring, and has no impact other than leaking memory during OOM. --- src/huge.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/huge.c b/src/huge.c index 7ad9b662..68839037 100644 --- a/src/huge.c +++ b/src/huge.c @@ -48,12 +48,8 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, */ is_zeroed = zero; arena = arena_choose(tsd, arena); - if (unlikely(arena == NULL)) { - base_node_dalloc(node); - return (NULL); - } - ret = arena_chunk_alloc_huge(arena, usize, alignment, &is_zeroed); - if (ret == NULL) { + if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena, + usize, alignment, &is_zeroed)) == NULL) { idalloct(tsd, node, try_tcache); return (NULL); } From a18c2b1f152b4334474ed32fc46d762d4fa54c2b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 5 Dec 2014 17:49:47 -0800 Subject: [PATCH 181/352] Style fixes. --- include/jemalloc/internal/atomic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h index a0488157..8b743b88 100644 --- a/include/jemalloc/internal/atomic.h +++ b/include/jemalloc/internal/atomic.h @@ -58,7 +58,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (__sync_sub_and_fetch(p, x)); } -#elif (defined(_MSC_VER)) +# elif (defined(_MSC_VER)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { @@ -72,7 +72,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x); } -#elif (defined(JEMALLOC_OSATOMIC)) +# elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { From 59cd80e6c6e36c26a880e86f6cde9f71808b256c Mon Sep 17 00:00:00 2001 From: Chih-hung Hsieh Date: Fri, 5 Dec 2014 17:42:41 -0800 Subject: [PATCH 182/352] Add a C11 atomics-based implementation of atomic.h API. --- configure.ac | 21 ++++++++++++++ include/jemalloc/internal/atomic.h | 28 +++++++++++++++++++ .../jemalloc/internal/jemalloc_internal.h.in | 4 +++ .../internal/jemalloc_internal_defs.h.in | 3 ++ 4 files changed, 56 insertions(+) diff --git a/configure.ac b/configure.ac index 5c51f27f..8b1e55e4 100644 --- a/configure.ac +++ b/configure.ac @@ -1199,6 +1199,27 @@ elif test "x${force_tls}" = "x1" ; then AC_MSG_ERROR([Failed to configure TLS, which is mandatory for correct function]) fi +dnl ============================================================================ +dnl Check for C11 atomics. + +JE_COMPILABLE([C11 atomics], [ +#include +#if (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_ATOMICS__) +#include +#else +#error Atomics not available +#endif +], [ + uint64_t *p = (uint64_t *)0; + uint64_t x = 1; + volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; + uint64_t r = atomic_fetch_add(a, x) + x; + return (r == 0); +], [je_cv_c11atomics]) +if test "x${je_cv_c11atomics}" = "xyes" ; then + AC_DEFINE([JEMALLOC_C11ATOMICS]) +fi + dnl ============================================================================ dnl Check for atomic(9) operations as provided on FreeBSD. diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h index 8b743b88..23ac93ff 100644 --- a/include/jemalloc/internal/atomic.h +++ b/include/jemalloc/internal/atomic.h @@ -72,6 +72,20 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x); } +# elif (defined(JEMALLOC_C11ATOMICS)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; + return (atomic_fetch_add(a, x) + x); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; + return (atomic_fetch_sub(a, x) - x); +} # elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) @@ -187,6 +201,20 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (InterlockedExchangeAdd(p, -((int32_t)x)) - x); } +# elif (defined(JEMALLOC_C11ATOMICS)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; + return (atomic_fetch_add(a, x) + x); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; + return (atomic_fetch_sub(a, x) - x); +} #elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 6f13093f..bf10617e 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -127,6 +127,10 @@ static const bool config_ivsalloc = #endif ; +#ifdef JEMALLOC_C11ATOMICS +#include +#endif + #ifdef JEMALLOC_ATOMIC9 #include #endif diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index dccbb1ed..2923e83f 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -22,6 +22,9 @@ */ #undef CPU_SPINWAIT +/* Defined if C11 atomics are available. */ +#undef JEMALLOC_C11ATOMICS + /* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */ #undef JEMALLOC_ATOMIC9 From e12eaf93dca308a426c182956197b0eeb5f2cff3 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 8 Dec 2014 14:40:14 -0800 Subject: [PATCH 183/352] Style and spelling fixes. --- include/jemalloc/internal/arena.h | 5 ++--- include/jemalloc/internal/extent.h | 2 +- include/jemalloc/internal/hash.h | 5 +++-- include/jemalloc/internal/jemalloc_internal.h.in | 2 +- .../jemalloc/internal/jemalloc_internal_decls.h | 2 +- .../jemalloc/internal/jemalloc_internal_defs.h.in | 14 ++++++-------- include/jemalloc/internal/ql.h | 4 +--- include/jemalloc/internal/qr.h | 6 ++++-- include/jemalloc/internal/rb.h | 4 ++-- include/jemalloc/internal/tcache.h | 2 +- include/jemalloc/internal/util.h | 6 +++--- src/arena.c | 2 +- src/chunk.c | 2 +- src/ckh.c | 4 ++-- src/jemalloc.c | 2 +- src/quarantine.c | 4 ++-- src/zone.c | 2 +- test/include/test/math.h | 2 +- test/include/test/thd.h | 2 +- test/include/test/timer.h | 4 +--- 20 files changed, 36 insertions(+), 40 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index a42522d6..1e190234 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -263,8 +263,7 @@ struct arena_s { /* * There are three classes of arena operations from a locking * perspective: - * 1) Thread asssignment (modifies nthreads) is protected by - * arenas_lock. + * 1) Thread assignment (modifies nthreads) is protected by arenas_lock. * 2) Bin-related operations are protected by bin locks. * 3) Chunk- and run-related operations are protected by this mutex. */ @@ -314,7 +313,7 @@ struct arena_s { arena_chunk_miscelms_t runs_dirty; /* - * user-configureable chunk allocation and deallocation functions. + * User-configurable chunk allocation and deallocation functions. */ chunk_alloc_t *chunk_alloc; chunk_dalloc_t *chunk_dalloc; diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 5b00076f..cbfc20a9 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -24,7 +24,7 @@ struct extent_node_s { /* Total region size. */ size_t size; - /* Arena from which this extent came, if any */ + /* Arena from which this extent came, if any. */ arena_t *arena; /* True if zero-filled; used by chunk recycling code. */ diff --git a/include/jemalloc/internal/hash.h b/include/jemalloc/internal/hash.h index a43bbbec..bcead337 100644 --- a/include/jemalloc/internal/hash.h +++ b/include/jemalloc/internal/hash.h @@ -35,13 +35,14 @@ JEMALLOC_INLINE uint32_t hash_rotl_32(uint32_t x, int8_t r) { - return (x << r) | (x >> (32 - r)); + return ((x << r) | (x >> (32 - r))); } JEMALLOC_INLINE uint64_t hash_rotl_64(uint64_t x, int8_t r) { - return (x << r) | (x >> (64 - r)); + + return ((x << r) | (x >> (64 - r))); } JEMALLOC_INLINE uint32_t diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index bf10617e..9bd501c0 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -302,7 +302,7 @@ typedef unsigned index_t; #define ALIGNMENT_CEILING(s, alignment) \ (((s) + (alignment - 1)) & (-(alignment))) -/* Declare a variable length array */ +/* Declare a variable-length array. */ #if __STDC_VERSION__ < 199901L # ifdef _MSC_VER # include diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h index fa590404..fb2effbf 100644 --- a/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -50,7 +50,7 @@ typedef intptr_t ssize_t; # define PATH_MAX 1024 # define STDERR_FILENO 2 # define __func__ __FUNCTION__ -/* Disable warnings about deprecated system functions */ +/* Disable warnings about deprecated system functions. */ # pragma warning(disable: 4996) #else # include diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 2923e83f..e172c661 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -38,7 +38,7 @@ * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the - * functions are defined in libgcc instead of being inlines) + * functions are defined in libgcc instead of being inlines). */ #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 @@ -46,7 +46,7 @@ * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the - * functions are defined in libgcc instead of being inlines) + * functions are defined in libgcc instead of being inlines). */ #undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 @@ -201,9 +201,7 @@ #undef JEMALLOC_PURGE_MADVISE_DONTNEED #undef JEMALLOC_PURGE_MADVISE_FREE -/* - * Define if operating system has alloca.h header. - */ +/* Define if operating system has alloca.h header. */ #undef JEMALLOC_HAS_ALLOCA_H /* C99 restrict keyword supported. */ @@ -221,13 +219,13 @@ /* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ #undef LG_SIZEOF_INTMAX_T -/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook) */ +/* glibc malloc hooks (__malloc_hook, __realloc_hook, __free_hook). */ #undef JEMALLOC_GLIBC_MALLOC_HOOK -/* glibc memalign hook */ +/* glibc memalign hook. */ #undef JEMALLOC_GLIBC_MEMALIGN_HOOK -/* adaptive mutex support in pthreads */ +/* Adaptive mutex support in pthreads. */ #undef JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/include/jemalloc/internal/ql.h b/include/jemalloc/internal/ql.h index f70c5f6f..1834bb85 100644 --- a/include/jemalloc/internal/ql.h +++ b/include/jemalloc/internal/ql.h @@ -1,6 +1,4 @@ -/* - * List definitions. - */ +/* List definitions. */ #define ql_head(a_type) \ struct { \ a_type *qlh_first; \ diff --git a/include/jemalloc/internal/qr.h b/include/jemalloc/internal/qr.h index 602944b9..0fbaec25 100644 --- a/include/jemalloc/internal/qr.h +++ b/include/jemalloc/internal/qr.h @@ -40,8 +40,10 @@ struct { \ (a_qr_b)->a_field.qre_prev = t; \ } while (0) -/* qr_meld() and qr_split() are functionally equivalent, so there's no need to - * have two copies of the code. */ +/* + * qr_meld() and qr_split() are functionally equivalent, so there's no need to + * have two copies of the code. + */ #define qr_split(a_qr_a, a_qr_b, a_field) \ qr_meld((a_qr_a), (a_qr_b), a_field) diff --git a/include/jemalloc/internal/rb.h b/include/jemalloc/internal/rb.h index 64fab89c..2ca8e593 100644 --- a/include/jemalloc/internal/rb.h +++ b/include/jemalloc/internal/rb.h @@ -200,7 +200,7 @@ a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ * int (a_cmp *)(a_type *a_node, a_type *a_other); * ^^^^^^ * or a_key - * Interpretation of comparision function return values: + * Interpretation of comparison function return values: * -1 : a_node < a_other * 0 : a_node == a_other * 1 : a_node > a_other @@ -693,7 +693,7 @@ a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \ rbtn_rotate_left(a_type, a_field, pathp->node, \ tnode); \ /* Balance restored, but rotation modified */\ - /* subree root, which may actually be the tree */\ + /* subtree root, which may actually be the tree */\ /* root. */\ if (pathp == path) { \ /* Set root. */ \ diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index fe9c47e8..3a3fd496 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -69,7 +69,7 @@ struct tcache_bin_s { struct tcache_s { ql_elm(tcache_t) link; /* Used for aggregating stats. */ - uint64_t prof_accumbytes;/* Cleared after arena_prof_accum() */ + uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */ arena_t *arena; /* This thread's arena. */ unsigned ev_cnt; /* Event count since incremental GC. */ index_t next_gc_bin; /* Next bin to GC. */ diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index 5af68329..b2b4ab74 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -127,7 +127,7 @@ int get_errno(void); #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) -/* Sanity check: */ +/* Sanity check. */ #if !defined(JEMALLOC_INTERNAL_FFSL) || !defined(JEMALLOC_INTERNAL_FFS) # error Both JEMALLOC_INTERNAL_FFSL && JEMALLOC_INTERNAL_FFS should have been defined by configure #endif @@ -231,7 +231,7 @@ lg_floor(size_t x) } #endif -/* Sets error code */ +/* Set error code. */ JEMALLOC_INLINE void set_errno(int errnum) { @@ -243,7 +243,7 @@ set_errno(int errnum) #endif } -/* Get last error code */ +/* Get last error code. */ JEMALLOC_INLINE int get_errno(void) { diff --git a/src/arena.c b/src/arena.c index f351c090..6f2410ac 100644 --- a/src/arena.c +++ b/src/arena.c @@ -36,7 +36,7 @@ arena_miscelm_to_bits(arena_chunk_map_misc_t *miscelm) arena_chunk_t *chunk = CHUNK_ADDR2BASE(miscelm); size_t pageind = arena_miscelm_to_pageind(miscelm); - return arena_mapbits_get(chunk, pageind); + return (arena_mapbits_get(chunk, pageind)); } JEMALLOC_INLINE_C int diff --git a/src/chunk.c b/src/chunk.c index b3737180..79264527 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -162,7 +162,7 @@ chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, new_addr, size, alignment, base, zero)) != NULL) return (ret); - /* requesting an address not implemented for chunk_alloc_mmap */ + /* Requesting an address not implemented for chunk_alloc_mmap(). */ if (new_addr == NULL && (ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) return (ret); diff --git a/src/ckh.c b/src/ckh.c index 3a545966..db2ae392 100644 --- a/src/ckh.c +++ b/src/ckh.c @@ -367,10 +367,10 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh->count = 0; /* - * Find the minimum power of 2 that is large enough to fit aBaseCount + * Find the minimum power of 2 that is large enough to fit minitems * entries. We are using (2+,2) cuckoo hashing, which has an expected * maximum load factor of at least ~0.86, so 0.75 is a conservative load - * factor that will typically allow 2^aLgMinItems to fit without ever + * factor that will typically allow mincells items to fit without ever * growing the table. */ assert(LG_CKH_BUCKET_CELLS > 0); diff --git a/src/jemalloc.c b/src/jemalloc.c index 8b2ab8d7..f7cc4575 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -808,7 +808,7 @@ malloc_conf_init(void) if (linklen == -1) { /* No configuration specified. */ linklen = 0; - /* restore errno */ + /* Restore errno. */ set_errno(saved_errno); } #endif diff --git a/src/quarantine.c b/src/quarantine.c index ddacc6ee..c5fa566b 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -2,7 +2,7 @@ #include "jemalloc/internal/jemalloc_internal.h" /* - * quarantine pointers close to NULL are used to encode state information that + * Quarantine pointers close to NULL are used to encode state information that * is used for cleaning up during thread shutdown. */ #define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) @@ -49,7 +49,7 @@ quarantine_alloc_hook_work(tsd_t *tsd) quarantine = quarantine_init(tsd, LG_MAXOBJS_INIT); /* * Check again whether quarantine has been initialized, because - * qurantine_init() may have triggered recursive initialization. + * quarantine_init() may have triggered recursive initialization. */ if (tsd_quarantine_get(tsd) == NULL) tsd_quarantine_set(tsd, quarantine); diff --git a/src/zone.c b/src/zone.c index c6bd533f..12e1734a 100644 --- a/src/zone.c +++ b/src/zone.c @@ -263,7 +263,7 @@ register_zone(void) * after the default zone. On OSX < 10.6, there is no purgeable * zone, so this does nothing. On OSX >= 10.6, unregistering * replaces the purgeable zone with the last registered zone - * above, i.e the default zone. Registering it again then puts + * above, i.e. the default zone. Registering it again then puts * it at the end, obviously after the default zone. */ if (purgeable_zone) { diff --git a/test/include/test/math.h b/test/include/test/math.h index a862ed7d..b057b29a 100644 --- a/test/include/test/math.h +++ b/test/include/test/math.h @@ -299,7 +299,7 @@ pt_chi2(double p, double df, double ln_gamma_df_2) /* * Given a value p in [0..1] and Gamma distribution shape and scale parameters, - * compute the upper limit on the definite integeral from [0..z] that satisfies + * compute the upper limit on the definite integral from [0..z] that satisfies * p. */ JEMALLOC_INLINE double diff --git a/test/include/test/thd.h b/test/include/test/thd.h index f941d7a7..47a51262 100644 --- a/test/include/test/thd.h +++ b/test/include/test/thd.h @@ -1,4 +1,4 @@ -/* Abstraction layer for threading in tests */ +/* Abstraction layer for threading in tests. */ #ifdef _WIN32 typedef HANDLE thd_t; #else diff --git a/test/include/test/timer.h b/test/include/test/timer.h index 6877e4ac..496072ac 100644 --- a/test/include/test/timer.h +++ b/test/include/test/timer.h @@ -1,6 +1,4 @@ -/* - * Simple timer, for use in benchmark reporting. - */ +/* Simple timer, for use in benchmark reporting. */ #include From b74041fb6e279bd8bbc133250241249f90cd619f Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Tue, 9 Dec 2014 17:41:34 -0500 Subject: [PATCH 184/352] Ignore MALLOC_CONF in set{uid,gid,cap} binaries. This eliminates the malloc tunables as tools for an attacker. Closes #173 --- configure.ac | 18 +++++++++++++++ .../internal/jemalloc_internal_defs.h.in | 10 ++++++++ src/jemalloc.c | 23 ++++++++++++++++++- 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 8b1e55e4..82bdefdd 100644 --- a/configure.ac +++ b/configure.ac @@ -1108,6 +1108,24 @@ fi CPPFLAGS="$CPPFLAGS -D_REENTRANT" +dnl Check if the GNU-specific secure_getenv function exists. +AC_CHECK_FUNC([secure_getenv], + [have_secure_getenv="1"], + [have_secure_getenv="0"] + ) +if test "x$have_secure_getenv" = "x1" ; then + AC_DEFINE([JEMALLOC_HAVE_SECURE_GETENV], [ ]) +fi + +dnl Check if the Solaris/BSD issetugid function exists. +AC_CHECK_FUNC([issetugid], + [have_issetugid="1"], + [have_issetugid="0"] + ) +if test "x$have_issetugid" = "x1" ; then + AC_DEFINE([JEMALLOC_HAVE_ISSETUGID], [ ]) +fi + dnl Check whether the BSD-specific _malloc_thread_cleanup() exists. If so, use dnl it rather than pthreads TSD cleanup functions to support cleanup during dnl thread exit, in order to avoid pthreads library recursion during diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index e172c661..c8d7dafb 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -66,6 +66,16 @@ */ #undef JEMALLOC_OSSPIN +/* + * Defined if secure_getenv(3) is available. + */ +#undef JEMALLOC_HAVE_SECURE_GETENV + +/* + * Defined if issetugid(2) is available. + */ +#undef JEMALLOC_HAVE_ISSETUGID + /* * Defined if _malloc_thread_cleanup() exists. At least in the case of * FreeBSD, pthread_key_create() allocates, which if used during malloc diff --git a/src/jemalloc.c b/src/jemalloc.c index f7cc4575..48de0da0 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -648,6 +648,27 @@ stats_print_atexit(void) * Begin initialization functions. */ +#ifndef JEMALLOC_HAVE_SECURE_GETENV +# ifdef JEMALLOC_HAVE_ISSETUGID +static char * +secure_getenv(const char *name) +{ + + if (issetugid() == 0) + return (getenv(name)); + else + return (NULL); +} +# else +static char * +secure_getenv(const char *name) +{ + + return (getenv(name)); +} +# endif +#endif + static unsigned malloc_ncpus(void) { @@ -824,7 +845,7 @@ malloc_conf_init(void) #endif ; - if ((opts = getenv(envname)) != NULL) { + if ((opts = secure_getenv(envname)) != NULL) { /* * Do nothing; opts is already initialized to * the value of the MALLOC_CONF environment From 2c5cb613dfbdf58f88152321b63e60c58cd23972 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Mon, 8 Dec 2014 19:12:41 -0200 Subject: [PATCH 185/352] Introduce two new modes of junk filling: "alloc" and "free". In addition to true/false, opt.junk can now be either "alloc" or "free", giving applications the possibility of junking memory only on allocation or deallocation. This resolves #172. --- Makefile.in | 2 + doc/jemalloc.xml.in | 20 ++++--- .../jemalloc/internal/jemalloc_internal.h.in | 4 +- include/jemalloc/internal/private_symbols.txt | 2 + include/jemalloc/internal/tcache.h | 10 ++-- src/arena.c | 55 ++++++++++--------- src/ctl.c | 2 +- src/huge.c | 12 ++-- src/jemalloc.c | 54 +++++++++++++++--- src/quarantine.c | 2 +- test/unit/junk.c | 41 +++++++++----- test/unit/junk_alloc.c | 3 + test/unit/junk_free.c | 3 + test/unit/mallctl.c | 2 +- 14 files changed, 140 insertions(+), 72 deletions(-) create mode 100644 test/unit/junk_alloc.c create mode 100644 test/unit/junk_free.c diff --git a/Makefile.in b/Makefile.in index 40644ce8..c268d002 100644 --- a/Makefile.in +++ b/Makefile.in @@ -118,6 +118,8 @@ TESTS_UNIT := $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/hash.c \ $(srcroot)test/unit/junk.c \ + $(srcroot)test/unit/junk_alloc.c \ + $(srcroot)test/unit/junk_free.c \ $(srcroot)test/unit/lg_chunk.c \ $(srcroot)test/unit/mallctl.c \ $(srcroot)test/unit/math.c \ diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 71b4cd19..0148f038 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -930,18 +930,20 @@ for (i = 0; i < nbins; i++) { opt.junk - (bool) + (const char *) r- [] - Junk filling enabled/disabled. If enabled, each byte - of uninitialized allocated memory will be initialized to - 0xa5. All deallocated memory will be initialized to - 0x5a. This is intended for debugging and will - impact performance negatively. This option is disabled by default - unless is specified during - configuration, in which case it is enabled by default unless running - inside Junk filling. If set to "alloc", each byte of + uninitialized allocated memory will be initialized to + 0xa5. If set to "free", all deallocated memory will + be initialized to 0x5a. If set to "true", both + allocated and deallocated memory will be initialized, and if set to + "false", junk filling be disabled entirely. This is intended for + debugging and will impact performance negatively. This option is + "false" by default unless is specified + during configuration, in which case it is "true" by default unless + running inside Valgrind. diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 9bd501c0..b7617dfd 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -376,7 +376,9 @@ typedef unsigned index_t; #define JEMALLOC_H_EXTERNS extern bool opt_abort; -extern bool opt_junk; +extern const char *opt_junk; +extern bool opt_junk_alloc; +extern bool opt_junk_free; extern size_t opt_quarantine; extern bool opt_redzone; extern bool opt_utrace; diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index ee973c9f..7e339152 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -274,6 +274,8 @@ nhbins opt_abort opt_dss opt_junk +opt_junk_alloc +opt_junk_free opt_lg_chunk opt_lg_dirty_mult opt_lg_prof_interval diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 3a3fd496..6e97b3dd 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -252,14 +252,14 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) if (likely(!zero)) { if (config_fill) { - if (unlikely(opt_junk)) { + if (unlikely(opt_junk_alloc)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], false); } else if (unlikely(opt_zero)) memset(ret, 0, usize); } } else { - if (config_fill && unlikely(opt_junk)) { + if (config_fill && unlikely(opt_junk_alloc)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } @@ -307,7 +307,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) } if (likely(!zero)) { if (config_fill) { - if (unlikely(opt_junk)) + if (unlikely(opt_junk_alloc)) memset(ret, 0xa5, usize); else if (unlikely(opt_zero)) memset(ret, 0, usize); @@ -333,7 +333,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr, index_t binind) assert(tcache_salloc(ptr) <= SMALL_MAXCLASS); - if (config_fill && unlikely(opt_junk)) + if (config_fill && unlikely(opt_junk_free)) arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); tbin = &tcache->tbins[binind]; @@ -362,7 +362,7 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) binind = size2index(size); - if (config_fill && unlikely(opt_junk)) + if (config_fill && unlikely(opt_junk_free)) arena_dalloc_junk_large(ptr, size); tbin = &tcache->tbins[binind]; diff --git a/src/arena.c b/src/arena.c index 6f2410ac..bf789950 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1450,7 +1450,7 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, } break; } - if (config_fill && unlikely(opt_junk)) { + if (config_fill && unlikely(opt_junk_alloc)) { arena_alloc_junk_small(ptr, &arena_bin_info[binind], true); } @@ -1512,24 +1512,27 @@ arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) size_t i; bool error = false; - for (i = 1; i <= redzone_size; i++) { - uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); - if (*byte != 0xa5) { - error = true; - arena_redzone_corruption(ptr, size, false, i, *byte); - if (reset) - *byte = 0xa5; - } - } - for (i = 0; i < redzone_size; i++) { - uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); - if (*byte != 0xa5) { - error = true; - arena_redzone_corruption(ptr, size, true, i, *byte); - if (reset) - *byte = 0xa5; + if (opt_junk_alloc) { + for (i = 1; i <= redzone_size; i++) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); + if (*byte != 0xa5) { + error = true; + arena_redzone_corruption(ptr, size, false, i, *byte); + if (reset) + *byte = 0xa5; + } + } + for (i = 0; i < redzone_size; i++) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); + if (*byte != 0xa5) { + error = true; + arena_redzone_corruption(ptr, size, true, i, *byte); + if (reset) + *byte = 0xa5; + } } } + if (opt_abort && error) abort(); } @@ -1560,7 +1563,7 @@ arena_quarantine_junk_small(void *ptr, size_t usize) index_t binind; arena_bin_info_t *bin_info; cassert(config_fill); - assert(opt_junk); + assert(opt_junk_free); assert(opt_quarantine); assert(usize <= SMALL_MAXCLASS); @@ -1604,7 +1607,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) if (!zero) { if (config_fill) { - if (unlikely(opt_junk)) { + if (unlikely(opt_junk_alloc)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], false); } else if (unlikely(opt_zero)) @@ -1612,7 +1615,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) } JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { - if (config_fill && unlikely(opt_junk)) { + if (config_fill && unlikely(opt_junk_alloc)) { arena_alloc_junk_small(ret, &arena_bin_info[binind], true); } @@ -1660,7 +1663,7 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) if (!zero) { if (config_fill) { - if (unlikely(opt_junk)) + if (unlikely(opt_junk_alloc)) memset(ret, 0xa5, usize); else if (unlikely(opt_zero)) memset(ret, 0, usize); @@ -1732,7 +1735,7 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) malloc_mutex_unlock(&arena->lock); if (config_fill && !zero) { - if (unlikely(opt_junk)) + if (unlikely(opt_junk_alloc)) memset(ret, 0xa5, size); else if (unlikely(opt_zero)) memset(ret, 0, size); @@ -1845,7 +1848,7 @@ arena_dalloc_bin_locked_impl(arena_t *arena, arena_chunk_t *chunk, void *ptr, bin = &arena->bins[binind]; bin_info = &arena_bin_info[binind]; - if (!junked && config_fill && unlikely(opt_junk)) + if (!junked && config_fill && unlikely(opt_junk_free)) arena_dalloc_junk_small(ptr, bin_info); arena_run_reg_dalloc(run, ptr); @@ -1908,7 +1911,7 @@ void arena_dalloc_junk_large(void *ptr, size_t usize) { - if (config_fill && unlikely(opt_junk)) + if (config_fill && unlikely(opt_junk_free)) memset(ptr, 0x5a, usize); } #ifdef JEMALLOC_JET @@ -2079,7 +2082,7 @@ static void arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) { - if (config_fill && unlikely(opt_junk)) { + if (config_fill && unlikely(opt_junk_free)) { memset((void *)((uintptr_t)ptr + usize), 0x5a, old_usize - usize); } @@ -2126,7 +2129,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, size, extra, zero); if (config_fill && !ret && !zero) { - if (unlikely(opt_junk)) { + if (unlikely(opt_junk_alloc)) { memset((void *)((uintptr_t)ptr + oldsize), 0xa5, isalloc(ptr, config_prof) - oldsize); diff --git a/src/ctl.c b/src/ctl.c index b367c9f6..90bad7ee 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -1234,7 +1234,7 @@ CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) -CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool) +CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, const char *) CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) diff --git a/src/huge.c b/src/huge.c index 68839037..416cb172 100644 --- a/src/huge.c +++ b/src/huge.c @@ -67,7 +67,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, if (zero || (config_fill && unlikely(opt_zero))) { if (!is_zeroed) memset(ret, 0, usize); - } else if (config_fill && unlikely(opt_junk)) + } else if (config_fill && unlikely(opt_junk_alloc)) memset(ret, 0xa5, usize); return (ret); @@ -81,7 +81,7 @@ static void huge_dalloc_junk(void *ptr, size_t usize) { - if (config_fill && have_dss && unlikely(opt_junk)) { + if (config_fill && have_dss && unlikely(opt_junk_free)) { /* * Only bother junk filling if the chunk isn't about to be * unmapped. @@ -117,7 +117,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, size_t sdiff = CHUNK_CEILING(usize) - usize; zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + usize), sdiff) : true; - if (config_fill && unlikely(opt_junk)) { + if (config_fill && unlikely(opt_junk_free)) { memset((void *)((uintptr_t)ptr + usize), 0x5a, oldsize - usize); zeroed = false; @@ -147,7 +147,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, memset((void *)((uintptr_t)ptr + oldsize), 0, usize - oldsize); } - } else if (config_fill && unlikely(opt_junk)) { + } else if (config_fill && unlikely(opt_junk_alloc)) { memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - oldsize); } @@ -165,7 +165,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) sdiff = CHUNK_CEILING(usize) - usize; zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + usize), sdiff) : true; - if (config_fill && unlikely(opt_junk)) { + if (config_fill && unlikely(opt_junk_free)) { huge_dalloc_junk((void *)((uintptr_t)ptr + usize), oldsize - usize); zeroed = false; @@ -234,7 +234,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { CHUNK_CEILING(oldsize)), 0, usize - CHUNK_CEILING(oldsize)); } - } else if (config_fill && unlikely(opt_junk)) { + } else if (config_fill && unlikely(opt_junk_alloc)) { memset((void *)((uintptr_t)ptr + oldsize), 0xa5, usize - oldsize); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 48de0da0..e63dab3e 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -13,13 +13,28 @@ bool opt_abort = false #endif ; -bool opt_junk = +const char *opt_junk = +#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) + "true" +#else + "false" +#endif + ; +bool opt_junk_alloc = #if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) true #else false #endif ; +bool opt_junk_free = +#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) + true +#else + false +#endif + ; + size_t opt_quarantine = ZU(0); bool opt_redzone = false; bool opt_utrace = false; @@ -784,7 +799,9 @@ malloc_conf_init(void) if (config_valgrind) { in_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; if (config_fill && unlikely(in_valgrind)) { - opt_junk = false; + opt_junk = "false"; + opt_junk_alloc = false; + opt_junk_free = false; assert(!opt_zero); opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; opt_redzone = true; @@ -867,13 +884,13 @@ malloc_conf_init(void) &vlen)) { #define CONF_MATCH(n) \ (sizeof(n)-1 == klen && strncmp(n, k, klen) == 0) +#define CONF_MATCH_VALUE(n) \ + (sizeof(n)-1 == vlen && strncmp(n, v, vlen) == 0) #define CONF_HANDLE_BOOL(o, n, cont) \ if (CONF_MATCH(n)) { \ - if (strncmp("true", v, vlen) == 0 && \ - vlen == sizeof("true")-1) \ + if (CONF_MATCH_VALUE("true")) \ o = true; \ - else if (strncmp("false", v, vlen) == \ - 0 && vlen == sizeof("false")-1) \ + else if (CONF_MATCH_VALUE("false")) \ o = false; \ else { \ malloc_conf_error( \ @@ -987,7 +1004,30 @@ malloc_conf_init(void) -1, (sizeof(size_t) << 3) - 1) CONF_HANDLE_BOOL(opt_stats_print, "stats_print", true) if (config_fill) { - CONF_HANDLE_BOOL(opt_junk, "junk", true) + if (CONF_MATCH("junk")) { + if (CONF_MATCH_VALUE("true")) { + opt_junk = "true"; + opt_junk_alloc = opt_junk_free = + true; + } else if (CONF_MATCH_VALUE("false")) { + opt_junk = "false"; + opt_junk_alloc = opt_junk_free = + false; + } else if (CONF_MATCH_VALUE("alloc")) { + opt_junk = "alloc"; + opt_junk_alloc = true; + opt_junk_free = false; + } else if (CONF_MATCH_VALUE("free")) { + opt_junk = "free"; + opt_junk_alloc = false; + opt_junk_free = true; + } else { + malloc_conf_error( + "Invalid conf value", k, + klen, v, vlen); + } + continue; + } CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", 0, SIZE_T_MAX, false) CONF_HANDLE_BOOL(opt_redzone, "redzone", true) diff --git a/src/quarantine.c b/src/quarantine.c index c5fa566b..12c37e0a 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -148,7 +148,7 @@ quarantine(tsd_t *tsd, void *ptr) obj->usize = usize; quarantine->curbytes += usize; quarantine->curobjs++; - if (config_fill && unlikely(opt_junk)) { + if (config_fill && unlikely(opt_junk_free)) { /* * Only do redzone validation if Valgrind isn't in * operation. diff --git a/test/unit/junk.c b/test/unit/junk.c index 1522a610..733f661e 100644 --- a/test/unit/junk.c +++ b/test/unit/junk.c @@ -1,8 +1,11 @@ #include "test/jemalloc_test.h" #ifdef JEMALLOC_FILL +# ifndef JEMALLOC_TEST_JUNK_OPT +# define JEMALLOC_TEST_JUNK_OPT "junk:true" +# endif const char *malloc_conf = - "abort:false,junk:true,zero:false,redzone:true,quarantine:0"; + "abort:false,zero:false,redzone:true,quarantine:0," JEMALLOC_TEST_JUNK_OPT; #endif static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig; @@ -69,12 +72,14 @@ test_junk(size_t sz_min, size_t sz_max) char *s; size_t sz_prev, sz, i; - arena_dalloc_junk_small_orig = arena_dalloc_junk_small; - arena_dalloc_junk_small = arena_dalloc_junk_small_intercept; - arena_dalloc_junk_large_orig = arena_dalloc_junk_large; - arena_dalloc_junk_large = arena_dalloc_junk_large_intercept; - huge_dalloc_junk_orig = huge_dalloc_junk; - huge_dalloc_junk = huge_dalloc_junk_intercept; + if (opt_junk_free) { + arena_dalloc_junk_small_orig = arena_dalloc_junk_small; + arena_dalloc_junk_small = arena_dalloc_junk_small_intercept; + arena_dalloc_junk_large_orig = arena_dalloc_junk_large; + arena_dalloc_junk_large = arena_dalloc_junk_large_intercept; + huge_dalloc_junk_orig = huge_dalloc_junk; + huge_dalloc_junk = huge_dalloc_junk_intercept; + } sz_prev = 0; s = (char *)mallocx(sz_min, 0); @@ -92,9 +97,11 @@ test_junk(size_t sz_min, size_t sz_max) } for (i = sz_prev; i < sz; i++) { - assert_c_eq(s[i], 0xa5, - "Newly allocated byte %zu/%zu isn't junk-filled", - i, sz); + if (opt_junk_alloc) { + assert_c_eq(s[i], 0xa5, + "Newly allocated byte %zu/%zu isn't " + "junk-filled", i, sz); + } s[i] = 'a'; } @@ -103,7 +110,7 @@ test_junk(size_t sz_min, size_t sz_max) s = (char *)rallocx(s, sz+1, 0); assert_ptr_not_null((void *)s, "Unexpected rallocx() failure"); - assert_true(saw_junking, + assert_true(!opt_junk_free || saw_junking, "Expected region of size %zu to be junk-filled", sz); } @@ -111,12 +118,14 @@ test_junk(size_t sz_min, size_t sz_max) watch_junking(s); dallocx(s, 0); - assert_true(saw_junking, + assert_true(!opt_junk_free || saw_junking, "Expected region of size %zu to be junk-filled", sz); - arena_dalloc_junk_small = arena_dalloc_junk_small_orig; - arena_dalloc_junk_large = arena_dalloc_junk_large_orig; - huge_dalloc_junk = huge_dalloc_junk_orig; + if (opt_junk_free) { + arena_dalloc_junk_small = arena_dalloc_junk_small_orig; + arena_dalloc_junk_large = arena_dalloc_junk_large_orig; + huge_dalloc_junk = huge_dalloc_junk_orig; + } } TEST_BEGIN(test_junk_small) @@ -204,6 +213,7 @@ TEST_BEGIN(test_junk_redzone) arena_redzone_corruption_t *arena_redzone_corruption_orig; test_skip_if(!config_fill); + test_skip_if(!opt_junk_alloc || !opt_junk_free); arena_redzone_corruption_orig = arena_redzone_corruption; arena_redzone_corruption = arena_redzone_corruption_replacement; @@ -234,6 +244,7 @@ int main(void) { + assert(opt_junk_alloc || opt_junk_free); return (test( test_junk_small, test_junk_large, diff --git a/test/unit/junk_alloc.c b/test/unit/junk_alloc.c new file mode 100644 index 00000000..8db3331d --- /dev/null +++ b/test/unit/junk_alloc.c @@ -0,0 +1,3 @@ +#define JEMALLOC_TEST_JUNK_OPT "junk:alloc" +#include "junk.c" +#undef JEMALLOC_TEST_JUNK_OPT diff --git a/test/unit/junk_free.c b/test/unit/junk_free.c new file mode 100644 index 00000000..482a61d0 --- /dev/null +++ b/test/unit/junk_free.c @@ -0,0 +1,3 @@ +#define JEMALLOC_TEST_JUNK_OPT "junk:free" +#include "junk.c" +#undef JEMALLOC_TEST_JUNK_OPT diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 028a9710..f4b7d1ab 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -164,7 +164,7 @@ TEST_BEGIN(test_mallctl_opt) TEST_MALLCTL_OPT(size_t, narenas, always); TEST_MALLCTL_OPT(ssize_t, lg_dirty_mult, always); TEST_MALLCTL_OPT(bool, stats_print, always); - TEST_MALLCTL_OPT(bool, junk, fill); + TEST_MALLCTL_OPT(const char *, junk, fill); TEST_MALLCTL_OPT(size_t, quarantine, fill); TEST_MALLCTL_OPT(bool, redzone, fill); TEST_MALLCTL_OPT(bool, zero, fill); From b4acf7300a4ca3423ca36fe227e9bc2e23f25b9f Mon Sep 17 00:00:00 2001 From: Bert Maher Date: Fri, 24 Oct 2014 14:09:42 -0700 Subject: [PATCH 186/352] [pprof] Produce global profile unless thread-local profile requested Currently pprof will print output for all threads if a single thread is not specified, but this doesn't play well with many output formats (e.g., any of the dot-based formats). Instead, default to printing just the overall profile when no specific thread is requested. This resolves #157. --- bin/pprof | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/pprof b/bin/pprof index 5a4c6cd7..df503aea 100755 --- a/bin/pprof +++ b/bin/pprof @@ -404,7 +404,7 @@ sub Init() { "edgefraction=f" => \$main::opt_edgefraction, "maxdegree=i" => \$main::opt_maxdegree, "focus=s" => \$main::opt_focus, - "thread=i" => \$main::opt_thread, + "thread=s" => \$main::opt_thread, "ignore=s" => \$main::opt_ignore, "scale=i" => \$main::opt_scale, "heapcheck" => \$main::opt_heapcheck, @@ -707,7 +707,8 @@ sub Main() { } if (defined($data->{threads})) { foreach my $thread (sort { $a <=> $b } keys(%{$data->{threads}})) { - if (!defined($main::opt_thread) || $main::opt_thread == $thread) { + if (defined($main::opt_thread) && + ($main::opt_thread eq '*' || $main::opt_thread == $thread)) { my $thread_profile = $data->{threads}{$thread}; FilterAndPrint($thread_profile, $symbols, $libs, $thread); } From 9c6a8d3b0cc14fd26b119ad08f190e537771464f Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Wed, 17 Dec 2014 14:46:35 -0200 Subject: [PATCH 187/352] Move variable declaration to the top its block for MSVC compatibility. --- src/arena.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index bf789950..1eb4000a 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2022,6 +2022,7 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, * following run, then merge the first part with the existing * allocation. */ + arena_run_t *run; size_t flag_dirty, splitsize, usize; usize = s2u(size + extra); @@ -2030,8 +2031,7 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, assert(usize >= usize_min); splitsize = usize - oldsize; - arena_run_t *run = &arena_miscelm_get(chunk, - pageind+npages)->run; + run = &arena_miscelm_get(chunk, pageind+npages)->run; arena_run_split_large(arena, run, splitsize, zero); size = oldsize + splitsize; From b7b44dfad09186cf74080818075eb0bfc0805e3b Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 18 Dec 2014 15:12:53 +0900 Subject: [PATCH 188/352] Make mixed declarations an error It often happens that code changes introduce mixed declarations, that then break building with Visual Studio. Since the code style is to not use mixed declarations anyways, we might as well enforce it with -Werror. --- configure.ac | 1 + 1 file changed, 1 insertion(+) diff --git a/configure.ac b/configure.ac index 82bdefdd..95133c49 100644 --- a/configure.ac +++ b/configure.ac @@ -134,6 +134,7 @@ if test "x$CFLAGS" = "x" ; then AC_DEFINE_UNQUOTED([JEMALLOC_HAS_RESTRICT]) fi JE_CFLAGS_APPEND([-Wall]) + JE_CFLAGS_APPEND([-Werror=declaration-after-statement]) JE_CFLAGS_APPEND([-pipe]) JE_CFLAGS_APPEND([-g3]) elif test "x$je_cv_msvc" = "xyes" ; then From 51f86346c000aa2a44abaab08caeedcb151e6556 Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Thu, 18 Dec 2014 15:01:21 +0900 Subject: [PATCH 189/352] Add a isblank definition for MSVC < 2013 --- include/jemalloc/internal/jemalloc_internal_decls.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/jemalloc/internal/jemalloc_internal_decls.h b/include/jemalloc/internal/jemalloc_internal_decls.h index fb2effbf..b10561c6 100644 --- a/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/include/jemalloc/internal/jemalloc_internal_decls.h @@ -52,6 +52,14 @@ typedef intptr_t ssize_t; # define __func__ __FUNCTION__ /* Disable warnings about deprecated system functions. */ # pragma warning(disable: 4996) +#if _MSC_VER < 1800 +static int +isblank(int c) +{ + + return (c == '\t' || c == ' '); +} +#endif #else # include #endif From 24057f3da8cd1b23955068a368165eba2eefb5c4 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 14 Jan 2015 16:27:31 -0800 Subject: [PATCH 190/352] Fix an infinite recursion bug related to a0/tsd bootstrapping. This resolves #184. --- src/chunk.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/chunk.c b/src/chunk.c index 79264527..b9a24416 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -254,7 +254,9 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, { arena_t *arena; - arena = arena_get(tsd_fetch(), arena_ind, false, true); + /* Dodge tsd for a0 in order to avoid bootstrapping issues. */ + arena = (arena_ind == 0) ? a0get() : arena_get(tsd_fetch(), arena_ind, + false, true); /* * The arena we're allocating on behalf of must have been initialized * already. From 44b57b8e8b25797b94c7cccc0e32705f76fcf03b Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 16 Jan 2015 18:04:17 -0800 Subject: [PATCH 191/352] Fix OOM handling in memalign() and valloc(). Fix memalign() and valloc() to heed imemalign()'s return value. Reported by Kurt Wampler. --- src/jemalloc.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index e63dab3e..aecdce34 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1751,7 +1751,8 @@ void * je_memalign(size_t alignment, size_t size) { void *ret JEMALLOC_CC_SILENCE_INIT(NULL); - imemalign(&ret, alignment, size, 1); + if (unlikely(imemalign(&ret, alignment, size, 1) != 0)) + ret = NULL; JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } @@ -1762,7 +1763,8 @@ void * je_valloc(size_t size) { void *ret JEMALLOC_CC_SILENCE_INIT(NULL); - imemalign(&ret, PAGE, size, 1); + if (unlikely(imemalign(&ret, PAGE, size, 1) != 0)) + ret = NULL; JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } From b617df81bbd35b2d7124b16df4024f9541644f6e Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Wed, 21 Jan 2015 15:02:42 -0500 Subject: [PATCH 192/352] Add missing symbols to private_symbols.txt. This resolves #185. --- include/jemalloc/internal/private_symbols.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 7e339152..39132b24 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -158,6 +158,7 @@ ctl_postfork_child ctl_postfork_parent ctl_prefork dss_prec_names +extent_tree_ad_empty extent_tree_ad_first extent_tree_ad_insert extent_tree_ad_iter @@ -174,6 +175,7 @@ extent_tree_ad_reverse_iter extent_tree_ad_reverse_iter_recurse extent_tree_ad_reverse_iter_start extent_tree_ad_search +extent_tree_szad_empty extent_tree_szad_first extent_tree_szad_insert extent_tree_szad_iter @@ -289,6 +291,7 @@ opt_prof_final opt_prof_gdump opt_prof_leak opt_prof_prefix +opt_prof_thread_active_init opt_quarantine opt_redzone opt_stats_print @@ -332,6 +335,7 @@ prof_tctx_set prof_tdata_cleanup prof_tdata_get prof_tdata_init +prof_tdata_reinit prof_thread_active_get prof_thread_active_init_get prof_thread_active_init_set From bc96876f99e89705817630b503ac54a5c48789ab Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 21 Jan 2015 09:01:43 -0800 Subject: [PATCH 193/352] Fix arenas_cache_cleanup(). Fix arenas_cache_cleanup() to check whether arenas_cache is NULL before deallocation, rather than checking arenas. --- src/jemalloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index aecdce34..c53129a5 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -601,7 +601,7 @@ arenas_cache_cleanup(tsd_t *tsd) arena_t **arenas_cache; arenas_cache = tsd_arenas_cache_get(tsd); - if (arenas != NULL) + if (arenas_cache != NULL) a0free(arenas_cache); } From 10aff3f3e1b8b3ac0348b259c439c9fe870a6b95 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 20 Jan 2015 15:37:51 -0800 Subject: [PATCH 194/352] Refactor bootstrapping to delay tsd initialization. Refactor bootstrapping to delay tsd initialization, primarily to support integration with FreeBSD's libc. Refactor a0*() for internal-only use, and add the bootstrap_{malloc,calloc,free}() API for use by FreeBSD's libc. This separation limits use of the a0*() functions to metadata allocation, which doesn't require malloc/calloc/free API compatibility. This resolves #170. --- .../jemalloc/internal/jemalloc_internal.h.in | 5 +- include/jemalloc/internal/private_symbols.txt | 6 +- include/jemalloc/internal/tsd.h | 2 +- src/ctl.c | 18 +- src/jemalloc.c | 307 +++++++++++------- src/tsd.c | 4 +- 6 files changed, 210 insertions(+), 132 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index b7617dfd..41078607 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -404,9 +404,8 @@ extern size_t const index2size_tab[NSIZES]; extern uint8_t const size2index_tab[]; arena_t *a0get(void); -void *a0malloc(size_t size); -void *a0calloc(size_t num, size_t size); -void a0free(void *ptr); +void *a0malloc(size_t size, bool zero); +void a0dalloc(void *ptr); arena_t *arenas_extend(unsigned ind); arena_t *arena_init(unsigned ind); unsigned narenas_total_get(void); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 39132b24..1aaf80b6 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -1,5 +1,4 @@ -a0calloc -a0free +a0dalloc a0get a0malloc arena_get @@ -107,6 +106,9 @@ bitmap_set bitmap_sfu bitmap_size bitmap_unset +bootstrap_calloc +bootstrap_free +bootstrap_malloc bt_init buferror chunk_alloc_arena diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index 35dd8628..dbb91a2e 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -48,7 +48,7 @@ typedef enum { * void example_tsd_set(example_t *val) {...} * * Note that all of the functions deal in terms of (a_type *) rather than - * (a_type) so that it is possible to support non-pointer types (unlike + * (a_type) so that it is possible to support non-pointer types (unlike * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is * cast to (void *). This means that the cleanup function needs to cast the * function argument to (a_type *), then dereference the resulting pointer to diff --git a/src/ctl.c b/src/ctl.c index 90bad7ee..6b95584b 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -484,14 +484,14 @@ ctl_arena_init(ctl_arena_stats_t *astats) if (astats->lstats == NULL) { astats->lstats = (malloc_large_stats_t *)a0malloc(nlclasses * - sizeof(malloc_large_stats_t)); + sizeof(malloc_large_stats_t), false); if (astats->lstats == NULL) return (true); } if (astats->hstats == NULL) { astats->hstats = (malloc_huge_stats_t *)a0malloc(nhclasses * - sizeof(malloc_huge_stats_t)); + sizeof(malloc_huge_stats_t), false); if (astats->hstats == NULL) return (true); } @@ -627,7 +627,7 @@ ctl_grow(void) /* Allocate extended arena stats. */ astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) * - sizeof(ctl_arena_stats_t)); + sizeof(ctl_arena_stats_t), false); if (astats == NULL) return (true); @@ -636,7 +636,7 @@ ctl_grow(void) sizeof(ctl_arena_stats_t)); memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) { - a0free(astats); + a0dalloc(astats); return (true); } /* Swap merged stats to their new location. */ @@ -649,7 +649,7 @@ ctl_grow(void) memcpy(&astats[ctl_stats.narenas + 1], &tstats, sizeof(ctl_arena_stats_t)); } - a0free(ctl_stats.arenas); + a0dalloc(ctl_stats.arenas); ctl_stats.arenas = astats; ctl_stats.narenas++; @@ -723,7 +723,7 @@ ctl_init(void) */ ctl_stats.narenas = narenas_total_get(); ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc( - (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); + (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t), false); if (ctl_stats.arenas == NULL) { ret = true; goto label_return; @@ -742,12 +742,12 @@ ctl_init(void) if (ctl_arena_init(&ctl_stats.arenas[i])) { unsigned j; for (j = 0; j < i; j++) { - a0free( + a0dalloc( ctl_stats.arenas[j].lstats); - a0free( + a0dalloc( ctl_stats.arenas[j].hstats); } - a0free(ctl_stats.arenas); + a0dalloc(ctl_stats.arenas); ctl_stats.arenas = NULL; ret = true; goto label_return; diff --git a/src/jemalloc.c b/src/jemalloc.c index c53129a5..632c8d3e 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -62,8 +62,13 @@ static unsigned narenas_total; static arena_t *a0; /* arenas[0]; read-only after initialization. */ static unsigned narenas_auto; /* Read-only after initialization. */ -/* Set to true once the allocator has been initialized. */ -static bool malloc_initialized = false; +typedef enum { + malloc_init_uninitialized = 3, + malloc_init_a0_initialized = 2, + malloc_init_recursible = 1, + malloc_init_initialized = 0 /* Common case --> jnz. */ +} malloc_init_t; +static malloc_init_t malloc_init_state = malloc_init_uninitialized; JEMALLOC_ALIGNED(CACHELINE) const size_t index2size_tab[NSIZES] = { @@ -218,6 +223,7 @@ typedef struct { * definition. */ +static bool malloc_init_hard_a0(void); static bool malloc_init_hard(void); /******************************************************************************/ @@ -225,6 +231,13 @@ static bool malloc_init_hard(void); * Begin miscellaneous support functions. */ +JEMALLOC_ALWAYS_INLINE_C bool +malloc_initialized(void) +{ + + return (malloc_init_state == malloc_init_initialized); +} + JEMALLOC_ALWAYS_INLINE_C void malloc_thread_init(void) { @@ -242,11 +255,20 @@ malloc_thread_init(void) quarantine_alloc_hook(); } +JEMALLOC_ALWAYS_INLINE_C bool +malloc_init_a0(void) +{ + + if (unlikely(malloc_init_state == malloc_init_uninitialized)) + return (malloc_init_hard_a0()); + return (false); +} + JEMALLOC_ALWAYS_INLINE_C bool malloc_init(void) { - if (unlikely(!malloc_initialized) && malloc_init_hard()) + if (unlikely(!malloc_initialized()) && malloc_init_hard()) return (true); malloc_thread_init(); @@ -254,10 +276,8 @@ malloc_init(void) } /* - * The a0*() functions are used instead of i[mcd]alloc() in bootstrap-sensitive - * situations that cannot tolerate TLS variable access. These functions are - * also exposed for use in static binaries on FreeBSD, hence the old-style - * malloc() API. + * The a0*() functions are used instead of i[mcd]alloc() in situations that + * cannot tolerate TLS variable access. */ arena_t * @@ -269,16 +289,13 @@ a0get(void) } static void * -a0alloc(size_t size, bool zero) +a0imalloc(size_t size, bool zero) { void *ret; - if (unlikely(malloc_init())) + if (unlikely(malloc_init_a0())) return (NULL); - if (size == 0) - size = 1; - if (likely(size <= arena_maxclass)) ret = arena_malloc(NULL, a0get(), size, zero, false); else @@ -287,28 +304,11 @@ a0alloc(size_t size, bool zero) return (ret); } -void * -a0malloc(size_t size) -{ - - return (a0alloc(size, false)); -} - -void * -a0calloc(size_t num, size_t size) -{ - - return (a0alloc(num * size, true)); -} - -void -a0free(void *ptr) +static void +a0idalloc(void *ptr) { arena_chunk_t *chunk; - if (ptr == NULL) - return; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) arena_dalloc(NULL, chunk, ptr, false); @@ -316,6 +316,60 @@ a0free(void *ptr) huge_dalloc(NULL, ptr, false); } +void * +a0malloc(size_t size, bool zero) +{ + + return (a0imalloc(size, zero)); +} + +void +a0dalloc(void *ptr) +{ + + a0idalloc(ptr); +} + +/* + * FreeBSD's libc uses the bootstrap_*() functions in bootstrap-senstive + * situations that cannot tolerate TLS variable access (TLS allocation and very + * early internal data structure initialization). + */ + +void * +bootstrap_malloc(size_t size) +{ + + if (unlikely(size == 0)) + size = 1; + + return (a0imalloc(size, false)); +} + +void * +bootstrap_calloc(size_t num, size_t size) +{ + size_t num_size; + + num_size = num * size; + if (unlikely(num_size == 0)) { + assert(num == 0 || size == 0); + num_size = 1; + } + + return (a0imalloc(num_size, true)); +} + +void +bootstrap_free(void *ptr) +{ + + if (unlikely(ptr == NULL)) + return; + + a0idalloc(ptr); +} + /* Create a new arena and insert it into the arenas array at index ind. */ static arena_t * arena_init_locked(unsigned ind) @@ -328,7 +382,7 @@ arena_init_locked(unsigned ind) unsigned narenas_new = narenas_total + 1; arena_t **arenas_new = (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * - sizeof(arena_t *))); + sizeof(arena_t *)), false); if (arenas_new == NULL) return (NULL); memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); @@ -338,7 +392,7 @@ arena_init_locked(unsigned ind) * base_alloc()). */ if (narenas_total != narenas_auto) - a0free(arenas); + a0dalloc(arenas); arenas = arenas_new; narenas_total = narenas_new; } @@ -449,7 +503,7 @@ arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) /* Deallocate old cache if it's too small. */ if (arenas_cache != NULL && narenas_cache < narenas_actual) { - a0free(arenas_cache); + a0dalloc(arenas_cache); arenas_cache = NULL; narenas_cache = 0; tsd_arenas_cache_set(tsd, arenas_cache); @@ -465,7 +519,7 @@ arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) if (!*arenas_cache_bypassp) { *arenas_cache_bypassp = true; arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * - narenas_cache); + narenas_cache, false); *arenas_cache_bypassp = false; } else arenas_cache = NULL; @@ -602,7 +656,7 @@ arenas_cache_cleanup(tsd_t *tsd) arenas_cache = tsd_arenas_cache_get(tsd); if (arenas_cache != NULL) - a0free(arenas_cache); + a0dalloc(arenas_cache); } void @@ -1091,19 +1145,18 @@ malloc_conf_init(void) } } +/* init_lock must be held. */ static bool -malloc_init_hard(void) +malloc_init_hard_needed(void) { - arena_t *init_arenas[1]; - malloc_mutex_lock(&init_lock); - if (malloc_initialized || IS_INITIALIZER) { + if (malloc_initialized() || (IS_INITIALIZER && malloc_init_state == + malloc_init_recursible)) { /* * Another thread initialized the allocator before this one * acquired init_lock, or this thread is the initializing * thread, and it is recursively allocating. */ - malloc_mutex_unlock(&init_lock); return (false); } #ifdef JEMALLOC_THREADED_INIT @@ -1113,23 +1166,23 @@ malloc_init_hard(void) malloc_mutex_unlock(&init_lock); CPU_SPINWAIT; malloc_mutex_lock(&init_lock); - } while (!malloc_initialized); - malloc_mutex_unlock(&init_lock); + } while (!malloc_initialized()); return (false); } #endif - malloc_initializer = INITIALIZER; + return (true); +} - if (malloc_tsd_boot0()) { - malloc_mutex_unlock(&init_lock); - return (true); - } +/* init_lock must be held. */ +static bool +malloc_init_hard_a0_locked(void) +{ + + malloc_initializer = INITIALIZER; if (config_prof) prof_boot0(); - malloc_conf_init(); - if (opt_stats_print) { /* Print statistics at exit. */ if (atexit(stats_print_atexit) != 0) { @@ -1138,68 +1191,60 @@ malloc_init_hard(void) abort(); } } - - if (base_boot()) { - malloc_mutex_unlock(&init_lock); + if (base_boot()) return (true); - } - - if (chunk_boot()) { - malloc_mutex_unlock(&init_lock); + if (chunk_boot()) return (true); - } - - if (ctl_boot()) { - malloc_mutex_unlock(&init_lock); + if (ctl_boot()) return (true); - } - if (config_prof) prof_boot1(); - arena_boot(); - - if (config_tcache && tcache_boot()) { - malloc_mutex_unlock(&init_lock); + if (config_tcache && tcache_boot()) return (true); - } - - if (huge_boot()) { - malloc_mutex_unlock(&init_lock); + if (huge_boot()) return (true); - } - - if (malloc_mutex_init(&arenas_lock)) { - malloc_mutex_unlock(&init_lock); + if (malloc_mutex_init(&arenas_lock)) return (true); - } - /* * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). */ narenas_total = narenas_auto = 1; - arenas = init_arenas; + arenas = &a0; memset(arenas, 0, sizeof(arena_t *) * narenas_auto); - /* * Initialize one arena here. The rest are lazily created in * arena_choose_hard(). */ - a0 = arena_init(0); - if (a0 == NULL) { - malloc_mutex_unlock(&init_lock); + if (arena_init(0) == NULL) return (true); - } + malloc_init_state = malloc_init_a0_initialized; + return (false); +} - if (config_prof && prof_boot2()) { - malloc_mutex_unlock(&init_lock); - return (true); - } +static bool +malloc_init_hard_a0(void) +{ + bool ret; + malloc_mutex_lock(&init_lock); + ret = malloc_init_hard_a0_locked(); + malloc_mutex_unlock(&init_lock); + return (ret); +} + +/* + * Initialize data structures which may trigger recursive allocation. + * + * init_lock must be held. + */ +static void +malloc_init_hard_recursible(void) +{ + + malloc_init_state = malloc_init_recursible; malloc_mutex_unlock(&init_lock); - /**********************************************************************/ - /* Recursive allocation may follow. */ ncpus = malloc_ncpus(); @@ -1213,15 +1258,16 @@ malloc_init_hard(void) abort(); } #endif - - /* Done recursively allocating. */ - /**********************************************************************/ malloc_mutex_lock(&init_lock); +} - if (mutex_boot()) { - malloc_mutex_unlock(&init_lock); +/* init_lock must be held. */ +static bool +malloc_init_hard_finish(void) +{ + + if (mutex_boot()) return (true); - } if (opt_narenas == 0) { /* @@ -1248,22 +1294,53 @@ malloc_init_hard(void) /* Allocate and initialize arenas. */ arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); - if (arenas == NULL) { - malloc_mutex_unlock(&init_lock); + if (arenas == NULL) return (true); - } /* * Zero the array. In practice, this should always be pre-zeroed, * since it was just mmap()ed, but let's be sure. */ memset(arenas, 0, sizeof(arena_t *) * narenas_total); /* Copy the pointer to the one arena that was already initialized. */ - arenas[0] = init_arenas[0]; + arenas[0] = a0; + + malloc_init_state = malloc_init_initialized; + return (false); +} + +static bool +malloc_init_hard(void) +{ + + malloc_mutex_lock(&init_lock); + if (!malloc_init_hard_needed()) { + malloc_mutex_unlock(&init_lock); + return (false); + } + + if (malloc_init_state != malloc_init_a0_initialized && + malloc_init_hard_a0_locked()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (malloc_tsd_boot0()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (config_prof && prof_boot2()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + malloc_init_hard_recursible(); + + if (malloc_init_hard_finish()) { + malloc_mutex_unlock(&init_lock); + return (true); + } - malloc_initialized = true; malloc_mutex_unlock(&init_lock); malloc_tsd_boot1(); - return (false); } @@ -1634,7 +1711,7 @@ ifree(tsd_t *tsd, void *ptr, bool try_tcache) UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); assert(ptr != NULL); - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); if (config_prof && opt_prof) { usize = isalloc(ptr, config_prof); @@ -1655,7 +1732,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache) UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); assert(ptr != NULL); - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); if (config_prof && opt_prof) prof_free(tsd, ptr, usize); @@ -1688,7 +1765,7 @@ je_realloc(void *ptr, size_t size) } if (likely(ptr != NULL)) { - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); @@ -2060,7 +2137,7 @@ je_rallocx(void *ptr, size_t size, int flags) assert(ptr != NULL); assert(size != 0); - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); @@ -2200,7 +2277,7 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) assert(ptr != NULL); assert(size != 0); assert(SIZE_T_MAX - size >= extra); - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); tsd = tsd_fetch(); @@ -2234,7 +2311,7 @@ je_sallocx(const void *ptr, int flags) { size_t usize; - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); if (config_ivsalloc) @@ -2254,7 +2331,7 @@ je_dallocx(void *ptr, int flags) bool try_tcache; assert(ptr != NULL); - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); tsd = tsd_fetch(); if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { @@ -2296,7 +2373,7 @@ je_sdallocx(void *ptr, size_t size, int flags) size_t usize; assert(ptr != NULL); - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); usize = inallocx(size, flags); assert(usize == isalloc(ptr, config_prof)); @@ -2375,7 +2452,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { size_t ret; - assert(malloc_initialized || IS_INITIALIZER); + assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); if (config_ivsalloc) @@ -2427,10 +2504,10 @@ _malloc_prefork(void) unsigned i; #ifdef JEMALLOC_MUTEX_INIT_CB - if (!malloc_initialized) + if (!malloc_initialized()) return; #endif - assert(malloc_initialized); + assert(malloc_initialized()); /* Acquire all mutexes in a safe order. */ ctl_prefork(); @@ -2456,10 +2533,10 @@ _malloc_postfork(void) unsigned i; #ifdef JEMALLOC_MUTEX_INIT_CB - if (!malloc_initialized) + if (!malloc_initialized()) return; #endif - assert(malloc_initialized); + assert(malloc_initialized()); /* Release all mutexes, now that fork() has completed. */ huge_postfork_parent(); @@ -2479,7 +2556,7 @@ jemalloc_postfork_child(void) { unsigned i; - assert(malloc_initialized); + assert(malloc_initialized()); /* Release all mutexes, now that fork() has completed. */ huge_postfork_child(); diff --git a/src/tsd.c b/src/tsd.c index 59253fe3..00d8f95f 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -15,14 +15,14 @@ void * malloc_tsd_malloc(size_t size) { - return (a0malloc(CACHELINE_CEILING(size))); + return (a0malloc(CACHELINE_CEILING(size), false)); } void malloc_tsd_dalloc(void *wrapper) { - a0free(wrapper); + a0dalloc(wrapper); } void From 228b2e92421d8cc7990e931e3144b6f1c3398501 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 22 Jan 2015 15:28:25 -0800 Subject: [PATCH 195/352] Document under what circumstances in-place resizing succeeds. This resolves #100. --- doc/jemalloc.xml.in | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 0148f038..858572d8 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -533,6 +533,22 @@ for (i = 0; i < nbins; i++) { nearest multiple of the cacheline size, or specify cacheline alignment when allocating. + The realloc, + rallocx, and + xallocx functions may resize allocations + without moving them under limited circumstances. Unlike the + *allocx API, the standard API does not + officially round up the usable size of an allocation to the nearest size + class, so technically it is necessary to call + realloc to grow e.g. a 9-byte allocation to + 16 bytes, or shrink a 16-byte allocation to 9 bytes. Growth and shrinkage + trivially succeeds in place as long as the pre-size and post-size both round + up to the same size class. No other API guarantees are made regarding + in-place resizing, but the current implementation also tries to resize large + and huge allocations in place, as long as the pre-size and post-size are + both large or both huge. In such cases shrinkage always succeeds, but + growth only succeeds if the trailing memory is currently available. + Assuming 4 MiB chunks, 4 KiB pages, and a 16-byte quantum on a 64-bit system, the size classes in each category are as shown in . From 8afcaa9d8133f0a147820799222492d1c251d285 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 22 Jan 2015 16:03:00 -0800 Subject: [PATCH 196/352] Update copyright dates for 2015. --- COPYING | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/COPYING b/COPYING index bdda0feb..611968cd 100644 --- a/COPYING +++ b/COPYING @@ -1,10 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2014 Jason Evans . +Copyright (C) 2002-2015 Jason Evans . All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2014 Facebook, Inc. All rights reserved. +Copyright (C) 2009-2015 Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From bec6a8da39e8cb7e59550541d429cff5e3dfb6d8 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 22 Jan 2015 17:55:58 -0800 Subject: [PATCH 197/352] Implement the jemalloc-config script. This resolves #133. --- .gitignore | 1 + Makefile.in | 3 +- bin/jemalloc-config.in | 79 ++++++++++++++++++++++++++++++++++++++++++ configure.ac | 10 ++++-- 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 bin/jemalloc-config.in diff --git a/.gitignore b/.gitignore index fd68315d..5cd3e922 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /*.gcov.* +/bin/jemalloc-config /bin/jemalloc.sh /config.stamp diff --git a/Makefile.in b/Makefile.in index c268d002..da397c38 100644 --- a/Makefile.in +++ b/Makefile.in @@ -73,7 +73,7 @@ endif LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix) # Lists of files. -BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh +BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \ $(srcroot)src/atomic.c $(srcroot)src/base.c $(srcroot)src/bitmap.c \ @@ -408,6 +408,7 @@ clean: rm -f $(objroot)*.gcov.* distclean: clean + rm -f $(objroot)bin/jemalloc-config rm -f $(objroot)bin/jemalloc.sh rm -f $(objroot)config.log rm -f $(objroot)config.status diff --git a/bin/jemalloc-config.in b/bin/jemalloc-config.in new file mode 100644 index 00000000..b016c8d3 --- /dev/null +++ b/bin/jemalloc-config.in @@ -0,0 +1,79 @@ +#!/bin/sh + +usage() { + cat < +Options: + --help | -h : Print usage. + --version : Print jemalloc version. + --revision : Print shared library revision number. + --config : Print configure options used to build jemalloc. + --prefix : Print installation directory prefix. + --bindir : Print binary installation directory. + --datadir : Print data installation directory. + --includedir : Print include installation directory. + --libdir : Print library installation directory. + --mandir : Print manual page installation directory. + --cc : Print compiler used to build jemalloc. + --cflags : Print compiler flags used to build jemalloc. + --cppflags : Print preprocessor flags used to build jemalloc. + --ldflags : Print library flags used to build jemalloc. + --libs : Print libraries jemalloc was linked against. +EOF +} + +prefix="@prefix@" +exec_prefix="@exec_prefix@" + +case "$1" in +--help | -h) + usage + exit 0 + ;; +--version) + echo "@jemalloc_version@" + ;; +--revision) + echo "@rev@" + ;; +--config) + echo "@CONFIG@" + ;; +--prefix) + echo "@PREFIX@" + ;; +--bindir) + echo "@BINDIR@" + ;; +--datadir) + echo "@DATADIR@" + ;; +--includedir) + echo "@INCLUDEDIR@" + ;; +--libdir) + echo "@LIBDIR@" + ;; +--mandir) + echo "@MANDIR@" + ;; +--cc) + echo "@CC@" + ;; +--cflags) + echo "@CFLAGS@" + ;; +--cppflags) + echo "@CPPFLAGS@" + ;; +--ldflags) + echo "@LDFLAGS@ @EXTRA_LDFLAGS@" + ;; +--libs) + echo "@LIBS@" + ;; +*) + usage + exit 1 +esac diff --git a/configure.ac b/configure.ac index 95133c49..0a4f01e8 100644 --- a/configure.ac +++ b/configure.ac @@ -43,6 +43,9 @@ AC_CACHE_CHECK([whether $1 is compilable], dnl ============================================================================ +CONFIG=`echo ${ac_configure_args} | sed -e "s#\'\([^ ]*\)\'#\1#g"` +AC_SUBST([CONFIG]) + dnl Library revision. rev=2 AC_SUBST([rev]) @@ -1585,7 +1588,7 @@ AC_CONFIG_HEADERS([$cfghdrs_tup]) dnl ============================================================================ dnl Generate outputs. -AC_CONFIG_FILES([$cfgoutputs_tup config.stamp bin/jemalloc.sh]) +AC_CONFIG_FILES([$cfgoutputs_tup config.stamp bin/jemalloc-config bin/jemalloc.sh]) AC_SUBST([cfgoutputs_in]) AC_SUBST([cfgoutputs_out]) AC_OUTPUT @@ -1596,9 +1599,10 @@ AC_MSG_RESULT([================================================================= AC_MSG_RESULT([jemalloc version : ${jemalloc_version}]) AC_MSG_RESULT([library revision : ${rev}]) AC_MSG_RESULT([]) +AC_MSG_RESULT([CONFIG : ${CONFIG}]) AC_MSG_RESULT([CC : ${CC}]) -AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) AC_MSG_RESULT([CFLAGS : ${CFLAGS}]) +AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) AC_MSG_RESULT([LDFLAGS : ${LDFLAGS}]) AC_MSG_RESULT([EXTRA_LDFLAGS : ${EXTRA_LDFLAGS}]) AC_MSG_RESULT([LIBS : ${LIBS}]) @@ -1609,9 +1613,9 @@ AC_MSG_RESULT([XSLROOT : ${XSLROOT}]) AC_MSG_RESULT([]) AC_MSG_RESULT([PREFIX : ${PREFIX}]) AC_MSG_RESULT([BINDIR : ${BINDIR}]) +AC_MSG_RESULT([DATADIR : ${DATADIR}]) AC_MSG_RESULT([INCLUDEDIR : ${INCLUDEDIR}]) AC_MSG_RESULT([LIBDIR : ${LIBDIR}]) -AC_MSG_RESULT([DATADIR : ${DATADIR}]) AC_MSG_RESULT([MANDIR : ${MANDIR}]) AC_MSG_RESULT([]) AC_MSG_RESULT([srcroot : ${srcroot}]) From ec98a44662a82aff30a54ed86bd9b24f36cfe67e Mon Sep 17 00:00:00 2001 From: Guilherme Goncalves Date: Fri, 23 Jan 2015 10:52:13 -0200 Subject: [PATCH 198/352] Use the correct type for opt.junk when printing stats. --- src/stats.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stats.c b/src/stats.c index 054f0332..2b3da645 100644 --- a/src/stats.c +++ b/src/stats.c @@ -461,7 +461,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, OPT_WRITE_SIZE_T(narenas) OPT_WRITE_SSIZE_T(lg_dirty_mult) OPT_WRITE_BOOL(stats_print) - OPT_WRITE_BOOL(junk) + OPT_WRITE_CHAR_P(junk) OPT_WRITE_SIZE_T(quarantine) OPT_WRITE_BOOL(redzone) OPT_WRITE_BOOL(zero) From 4581b97809e7e545c38b996870a4e7284a620bc5 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 27 Nov 2014 17:22:36 -0200 Subject: [PATCH 199/352] Implement metadata statistics. There are three categories of metadata: - Base allocations are used for bootstrap-sensitive internal allocator data structures. - Arena chunk headers comprise pages which track the states of the non-metadata pages. - Internal allocations differ from application-originated allocations in that they are for internal use, and that they are omitted from heap profiles. The metadata statistics comprise the metadata categories as follows: - stats.metadata: All metadata -- base + arena chunk headers + internal allocations. - stats.arenas..metadata.mapped: Arena chunk headers. - stats.arenas..metadata.allocated: Internal allocations. This is reported separately from the other metadata statistics because it overlaps with the allocated and active statistics, whereas the other metadata statistics do not. Base allocations are not reported separately, though their magnitude can be computed by subtracting the arena-specific metadata. This resolves #163. --- doc/jemalloc.xml.in | 47 ++++ include/jemalloc/internal/arena.h | 34 +++ include/jemalloc/internal/base.h | 1 + include/jemalloc/internal/ctl.h | 1 + include/jemalloc/internal/huge.h | 1 + .../jemalloc/internal/jemalloc_internal.h.in | 230 +++++++++++------- include/jemalloc/internal/private_symbols.txt | 10 + include/jemalloc/internal/stats.h | 7 + src/arena.c | 10 +- src/base.c | 15 ++ src/ctl.c | 30 ++- src/huge.c | 113 ++++----- src/jemalloc.c | 37 ++- src/prof.c | 30 +-- src/quarantine.c | 11 +- src/stats.c | 14 +- src/tcache.c | 4 +- src/tsd.c | 2 +- 18 files changed, 393 insertions(+), 204 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 858572d8..08fd4eb3 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1773,6 +1773,21 @@ malloc_conf = "xmalloc:true";]]> entirely devoted to allocator metadata. + + + stats.metadata + (size_t) + r- + [] + + Total number of bytes dedicated to metadata, which + comprise base allocations used for bootstrap-sensitive internal + allocator data structures, arena chunk headers (see stats.arenas.<i>.metadata.mapped), + and internal allocations (see stats.arenas.<i>.metadata.allocated). + + stats.mapped @@ -1875,6 +1890,38 @@ malloc_conf = "xmalloc:true";]]> Number of mapped bytes. + + + stats.arenas.<i>.metadata.mapped + (size_t) + r- + [] + + Number of mapped bytes in arena chunk headers, which + track the states of the non-metadata pages. + + + + + stats.arenas.<i>.metadata.allocated + (size_t) + r- + [] + + Number of bytes dedicated to internal allocations. + Internal allocations differ from application-originated allocations in + that they are for internal use, and that they are omitted from heap + profiles. This statistic is reported separately from stats.metadata and + stats.arenas.<i>.metadata.mapped + because it overlaps with e.g. the stats.allocated and + stats.active + statistics, whereas the other metadata statistics do + not. + + stats.arenas.<i>.npurge diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 1e190234..46367f68 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -437,6 +437,9 @@ void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, index_t binind, size_t flags); void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, size_t unzeroed); +void arena_metadata_allocated_add(arena_t *arena, size_t size); +void arena_metadata_allocated_sub(arena_t *arena, size_t size); +size_t arena_metadata_allocated_get(arena_t *arena); bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); bool arena_prof_accum(arena_t *arena, uint64_t accumbytes); @@ -448,6 +451,7 @@ prof_tctx_t *arena_prof_tctx_get(const void *ptr); void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, bool try_tcache); +arena_t *arena_aalloc(const void *ptr); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache); @@ -699,6 +703,27 @@ arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, unzeroed); } +JEMALLOC_INLINE void +arena_metadata_allocated_add(arena_t *arena, size_t size) +{ + + atomic_add_z(&arena->stats.metadata_allocated, size); +} + +JEMALLOC_INLINE void +arena_metadata_allocated_sub(arena_t *arena, size_t size) +{ + + atomic_sub_z(&arena->stats.metadata_allocated, size); +} + +JEMALLOC_INLINE size_t +arena_metadata_allocated_get(arena_t *arena) +{ + + return (atomic_read_z(&arena->stats.metadata_allocated)); +} + JEMALLOC_INLINE bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) { @@ -952,6 +977,15 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, } } +JEMALLOC_ALWAYS_INLINE arena_t * +arena_aalloc(const void *ptr) +{ + arena_chunk_t *chunk; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + return (chunk->arena); +} + /* Return the size of the allocation pointed to by ptr. */ JEMALLOC_ALWAYS_INLINE size_t arena_salloc(const void *ptr, bool demote) diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index 3fb80b92..18b7a72d 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -13,6 +13,7 @@ void *base_alloc(size_t size); void *base_calloc(size_t number, size_t size); extent_node_t *base_node_alloc(void); void base_node_dalloc(extent_node_t *node); +size_t base_allocated_get(void); bool base_boot(void); void base_prefork(void); void base_postfork_parent(void); diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index a3e899ea..65617bc9 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -52,6 +52,7 @@ struct ctl_arena_stats_s { struct ctl_stats_s { size_t allocated; size_t active; + size_t metadata; size_t mapped; struct { size_t current; /* stats_chunks.curchunks */ diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 39d8aa50..decb0249 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -23,6 +23,7 @@ typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; #endif void huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache); +arena_t *huge_aalloc(const void *ptr); size_t huge_salloc(const void *ptr); prof_tctx_t *huge_prof_tctx_get(const void *ptr); void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 41078607..a4778550 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -404,8 +404,9 @@ extern size_t const index2size_tab[NSIZES]; extern uint8_t const size2index_tab[]; arena_t *a0get(void); -void *a0malloc(size_t size, bool zero); +void *a0malloc(size_t size); void a0dalloc(void *ptr); +size_t a0allocated(void); arena_t *arenas_extend(unsigned ind); arena_t *arena_init(unsigned ind); unsigned narenas_total_get(void); @@ -776,21 +777,27 @@ arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, #include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE +arena_t *iaalloc(const void *ptr); +size_t isalloc(const void *ptr, bool demote); +void *iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, + bool is_metadata, arena_t *arena); void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); void *imalloc(tsd_t *tsd, size_t size); void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); void *icalloc(tsd_t *tsd, size_t size); +void *ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, + bool try_tcache, bool is_metadata, arena_t *arena); void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena); void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); -size_t isalloc(const void *ptr, bool demote); size_t ivsalloc(const void *ptr, bool demote); size_t u2rz(size_t usize); size_t p2rz(const void *ptr); +void idalloctm(tsd_t *tsd, void *ptr, bool try_tcache, bool is_metadata); void idalloct(tsd_t *tsd, void *ptr, bool try_tcache); -void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); void idalloc(tsd_t *tsd, void *ptr); void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache); +void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, @@ -805,76 +812,21 @@ bool ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -JEMALLOC_ALWAYS_INLINE void * -imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) +JEMALLOC_ALWAYS_INLINE arena_t * +iaalloc(const void *ptr) { + arena_t *arena; + arena_chunk_t *chunk; - assert(size != 0); + assert(ptr != NULL); - if (likely(size <= arena_maxclass)) - return (arena_malloc(tsd, arena, size, false, try_tcache)); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (likely(chunk != ptr)) + arena = arena_aalloc(ptr); else - return (huge_malloc(tsd, arena, size, false, try_tcache)); -} + arena = huge_aalloc(ptr); -JEMALLOC_ALWAYS_INLINE void * -imalloc(tsd_t *tsd, size_t size) -{ - - return (imalloct(tsd, size, true, NULL)); -} - -JEMALLOC_ALWAYS_INLINE void * -icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) -{ - - if (likely(size <= arena_maxclass)) - return (arena_malloc(tsd, arena, size, true, try_tcache)); - else - return (huge_malloc(tsd, arena, size, true, try_tcache)); -} - -JEMALLOC_ALWAYS_INLINE void * -icalloc(tsd_t *tsd, size_t size) -{ - - return (icalloct(tsd, size, true, NULL)); -} - -JEMALLOC_ALWAYS_INLINE void * -ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena) -{ - void *ret; - - assert(usize != 0); - assert(usize == sa2u(usize, alignment)); - - if (usize <= SMALL_MAXCLASS && alignment < PAGE) - ret = arena_malloc(tsd, arena, usize, zero, try_tcache); - else { - if (likely(usize <= arena_maxclass)) { - arena = arena_choose(tsd, arena); - if (unlikely(arena == NULL)) - return (NULL); - ret = arena_palloc(arena, usize, alignment, zero); - } else if (likely(alignment <= chunksize)) - ret = huge_malloc(tsd, arena, usize, zero, try_tcache); - else { - ret = huge_palloc(tsd, arena, usize, alignment, zero, - try_tcache); - } - } - - assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); - return (ret); -} - -JEMALLOC_ALWAYS_INLINE void * -ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) -{ - - return (ipalloct(tsd, usize, alignment, zero, true, NULL)); + return (arena); } /* @@ -901,6 +853,101 @@ isalloc(const void *ptr, bool demote) return (ret); } +JEMALLOC_ALWAYS_INLINE void * +iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, bool is_metadata, + arena_t *arena) +{ + void *ret; + + assert(size != 0); + + if (likely(size <= arena_maxclass)) + ret = arena_malloc(tsd, arena, size, zero, try_tcache); + else + ret = huge_malloc(tsd, arena, size, zero, try_tcache); + if (config_stats && is_metadata && likely(ret != NULL)) { + arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, + config_prof)); + } + return (ret); +} + +JEMALLOC_ALWAYS_INLINE void * +imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) +{ + + return (iallocztm(tsd, size, false, try_tcache, false, arena)); +} + +JEMALLOC_ALWAYS_INLINE void * +imalloc(tsd_t *tsd, size_t size) +{ + + return (iallocztm(tsd, size, false, true, false, NULL)); +} + +JEMALLOC_ALWAYS_INLINE void * +icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) +{ + + return (iallocztm(tsd, size, true, try_tcache, false, arena)); +} + +JEMALLOC_ALWAYS_INLINE void * +icalloc(tsd_t *tsd, size_t size) +{ + + return (iallocztm(tsd, size, true, true, false, NULL)); +} + +JEMALLOC_ALWAYS_INLINE void * +ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, + bool try_tcache, bool is_metadata, arena_t *arena) +{ + void *ret; + + assert(usize != 0); + assert(usize == sa2u(usize, alignment)); + + if (usize <= SMALL_MAXCLASS && alignment < PAGE) + ret = arena_malloc(tsd, arena, usize, zero, try_tcache); + else { + if (likely(usize <= arena_maxclass)) { + arena = arena_choose(tsd, arena); + if (unlikely(arena == NULL)) + return (NULL); + ret = arena_palloc(arena, usize, alignment, zero); + } else if (likely(alignment <= chunksize)) + ret = huge_malloc(tsd, arena, usize, zero, try_tcache); + else { + ret = huge_palloc(tsd, arena, usize, alignment, zero, + try_tcache); + } + } + assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); + if (config_stats && is_metadata && likely(ret != NULL)) { + arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, + config_prof)); + } + return (ret); +} + +JEMALLOC_ALWAYS_INLINE void * +ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena) +{ + + return (ipallocztm(tsd, usize, alignment, zero, try_tcache, false, + arena)); +} + +JEMALLOC_ALWAYS_INLINE void * +ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) +{ + + return (ipallocztm(tsd, usize, alignment, zero, true, false, NULL)); +} + JEMALLOC_ALWAYS_INLINE size_t ivsalloc(const void *ptr, bool demote) { @@ -935,11 +982,15 @@ p2rz(const void *ptr) } JEMALLOC_ALWAYS_INLINE void -idalloct(tsd_t *tsd, void *ptr, bool try_tcache) +idalloctm(tsd_t *tsd, void *ptr, bool try_tcache, bool is_metadata) { arena_chunk_t *chunk; assert(ptr != NULL); + if (config_stats && is_metadata) { + arena_metadata_allocated_sub(iaalloc(ptr), isalloc(ptr, + config_prof)); + } chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) @@ -948,6 +999,30 @@ idalloct(tsd_t *tsd, void *ptr, bool try_tcache) huge_dalloc(tsd, ptr, try_tcache); } +JEMALLOC_ALWAYS_INLINE void +idalloct(tsd_t *tsd, void *ptr, bool try_tcache) +{ + + idalloctm(tsd, ptr, try_tcache, false); +} + +JEMALLOC_ALWAYS_INLINE void +idalloc(tsd_t *tsd, void *ptr) +{ + + idalloctm(tsd, ptr, true, false); +} + +JEMALLOC_ALWAYS_INLINE void +iqalloc(tsd_t *tsd, void *ptr, bool try_tcache) +{ + + if (config_fill && unlikely(opt_quarantine)) + quarantine(tsd, ptr); + else + idalloctm(tsd, ptr, try_tcache, false); +} + JEMALLOC_ALWAYS_INLINE void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) { @@ -962,23 +1037,6 @@ isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) huge_dalloc(tsd, ptr, try_tcache); } -JEMALLOC_ALWAYS_INLINE void -idalloc(tsd_t *tsd, void *ptr) -{ - - idalloct(tsd, ptr, true); -} - -JEMALLOC_ALWAYS_INLINE void -iqalloc(tsd_t *tsd, void *ptr, bool try_tcache) -{ - - if (config_fill && unlikely(opt_quarantine)) - quarantine(tsd, ptr); - else - idalloct(tsd, ptr, try_tcache); -} - JEMALLOC_ALWAYS_INLINE void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) { diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 1aaf80b6..dfa87551 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -1,6 +1,7 @@ a0dalloc a0get a0malloc +arena_aalloc arena_get arena_get_hard arena_alloc_junk_small @@ -50,6 +51,9 @@ arena_mapbitsp_read arena_mapbitsp_write arena_maxclass arena_maxrun +arena_metadata_allocated_add +arena_metadata_allocated_get +arena_metadata_allocated_sub arena_migrate arena_miscelm_get arena_miscelm_to_pageind @@ -90,6 +94,7 @@ atomic_sub_uint32 atomic_sub_uint64 atomic_sub_z base_alloc +base_allocated_get base_boot base_calloc base_node_alloc @@ -205,6 +210,7 @@ hash_rotl_64 hash_x64_128 hash_x86_128 hash_x86_32 +huge_aalloc huge_allocated huge_boot huge_dalloc @@ -221,10 +227,13 @@ huge_prof_tctx_set huge_ralloc huge_ralloc_no_move huge_salloc +iaalloc +iallocztm icalloc icalloct idalloc idalloct +idalloctm imalloc imalloct in_valgrind @@ -234,6 +243,7 @@ index2size_lookup index2size_tab ipalloc ipalloct +ipallocztm iqalloc iralloc iralloct diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index d8600ed4..7cba77b9 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -111,6 +111,13 @@ struct arena_stats_s { uint64_t nmadvise; uint64_t purged; + /* + * Number of bytes currently mapped purely for metadata purposes, and + * number of bytes currently allocated for internal metadata. + */ + size_t metadata_mapped; + size_t metadata_allocated; /* Protected via atomic_*_z(). */ + /* Per-size-category statistics. */ size_t allocated_large; uint64_t nmalloc_large; diff --git a/src/arena.c b/src/arena.c index 1eb4000a..984b8ad2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -405,8 +405,10 @@ arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL, size, alignment, zero); malloc_mutex_lock(&arena->lock); - if (config_stats && chunk != NULL) + if (config_stats && chunk != NULL) { arena->stats.mapped += chunksize; + arena->stats.metadata_mapped += (map_bias << LG_PAGE); + } return (chunk); } @@ -514,8 +516,10 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) malloc_mutex_unlock(&arena->lock); chunk_dalloc((void *)spare, chunksize, arena->ind); malloc_mutex_lock(&arena->lock); - if (config_stats) + if (config_stats) { arena->stats.mapped -= chunksize; + arena->stats.metadata_mapped -= (map_bias << LG_PAGE); + } } else arena->spare = chunk; } @@ -2273,6 +2277,8 @@ arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, astats->npurge += arena->stats.npurge; astats->nmadvise += arena->stats.nmadvise; astats->purged += arena->stats.purged; + astats->metadata_mapped += arena->stats.metadata_mapped; + astats->metadata_allocated += arena_metadata_allocated_get(arena); astats->allocated_large += arena->stats.allocated_large; astats->nmalloc_large += arena->stats.nmalloc_large; astats->ndalloc_large += arena->stats.ndalloc_large; diff --git a/src/base.c b/src/base.c index 409c7bb7..22f36139 100644 --- a/src/base.c +++ b/src/base.c @@ -16,6 +16,8 @@ static void *base_next_addr; static void *base_past_addr; /* Addr immediately past base_pages. */ static extent_node_t *base_nodes; +static size_t base_allocated; + /******************************************************************************/ static bool @@ -54,6 +56,8 @@ base_alloc(size_t size) /* Allocate. */ ret = base_next_addr; base_next_addr = (void *)((uintptr_t)base_next_addr + csize); + if (config_stats) + base_allocated += csize; malloc_mutex_unlock(&base_mtx); JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); @@ -102,6 +106,17 @@ base_node_dalloc(extent_node_t *node) malloc_mutex_unlock(&base_mtx); } +size_t +base_allocated_get(void) +{ + size_t ret; + + malloc_mutex_lock(&base_mtx); + ret = base_allocated; + malloc_mutex_unlock(&base_mtx); + return (ret); +} + bool base_boot(void) { diff --git a/src/ctl.c b/src/ctl.c index 6b95584b..b65af520 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -183,10 +183,13 @@ CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_npurge) CTL_PROTO(stats_arenas_i_nmadvise) CTL_PROTO(stats_arenas_i_purged) +CTL_PROTO(stats_arenas_i_metadata_mapped) +CTL_PROTO(stats_arenas_i_metadata_allocated) INDEX_PROTO(stats_arenas_i) CTL_PROTO(stats_cactive) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) +CTL_PROTO(stats_metadata) CTL_PROTO(stats_mapped) /******************************************************************************/ @@ -355,6 +358,11 @@ static const ctl_named_node_t stats_chunks_node[] = { {NAME("high"), CTL(stats_chunks_high)} }; +static const ctl_named_node_t stats_arenas_i_metadata_node[] = { + {NAME("mapped"), CTL(stats_arenas_i_metadata_mapped)}, + {NAME("allocated"), CTL(stats_arenas_i_metadata_allocated)} +}; + static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, @@ -432,6 +440,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("npurge"), CTL(stats_arenas_i_npurge)}, {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, {NAME("purged"), CTL(stats_arenas_i_purged)}, + {NAME("metadata"), CHILD(named, stats_arenas_i_metadata)}, {NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("huge"), CHILD(named, stats_arenas_i_huge)}, @@ -451,6 +460,7 @@ static const ctl_named_node_t stats_node[] = { {NAME("cactive"), CTL(stats_cactive)}, {NAME("allocated"), CTL(stats_allocated)}, {NAME("active"), CTL(stats_active)}, + {NAME("metadata"), CTL(stats_metadata)}, {NAME("mapped"), CTL(stats_mapped)}, {NAME("chunks"), CHILD(named, stats_chunks)}, {NAME("arenas"), CHILD(indexed, stats_arenas)} @@ -484,14 +494,14 @@ ctl_arena_init(ctl_arena_stats_t *astats) if (astats->lstats == NULL) { astats->lstats = (malloc_large_stats_t *)a0malloc(nlclasses * - sizeof(malloc_large_stats_t), false); + sizeof(malloc_large_stats_t)); if (astats->lstats == NULL) return (true); } if (astats->hstats == NULL) { astats->hstats = (malloc_huge_stats_t *)a0malloc(nhclasses * - sizeof(malloc_huge_stats_t), false); + sizeof(malloc_huge_stats_t)); if (astats->hstats == NULL) return (true); } @@ -551,6 +561,9 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->astats.nmadvise += astats->astats.nmadvise; sstats->astats.purged += astats->astats.purged; + sstats->astats.metadata_mapped += astats->astats.metadata_mapped; + sstats->astats.metadata_allocated += astats->astats.metadata_allocated; + sstats->allocated_small += astats->allocated_small; sstats->nmalloc_small += astats->nmalloc_small; sstats->ndalloc_small += astats->ndalloc_small; @@ -627,7 +640,7 @@ ctl_grow(void) /* Allocate extended arena stats. */ astats = (ctl_arena_stats_t *)a0malloc((ctl_stats.narenas + 2) * - sizeof(ctl_arena_stats_t), false); + sizeof(ctl_arena_stats_t)); if (astats == NULL) return (true); @@ -704,6 +717,10 @@ ctl_refresh(void) + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge; ctl_stats.active = (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE); + ctl_stats.metadata = base_allocated_get() + + ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped + + ctl_stats.arenas[ctl_stats.narenas].astats + .metadata_allocated; ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); } @@ -723,7 +740,7 @@ ctl_init(void) */ ctl_stats.narenas = narenas_total_get(); ctl_stats.arenas = (ctl_arena_stats_t *)a0malloc( - (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t), false); + (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); if (ctl_stats.arenas == NULL) { ret = true; goto label_return; @@ -1806,6 +1823,7 @@ CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) +CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current, @@ -1825,6 +1843,10 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_purged, ctl_stats.arenas[mib[2]].astats.purged, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_mapped, + ctl_stats.arenas[mib[2]].astats.metadata_mapped, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_allocated, + ctl_stats.arenas[mib[2]].astats.metadata_allocated, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, ctl_stats.arenas[mib[2]].allocated_small, size_t) diff --git a/src/huge.c b/src/huge.c index 416cb172..c4d1ebc6 100644 --- a/src/huge.c +++ b/src/huge.c @@ -37,8 +37,8 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, /* Allocate one or more contiguous chunks for this request. */ /* Allocate an extent node with which to track the chunk. */ - node = ipalloct(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), - CACHELINE, false, try_tcache, NULL); + node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), + CACHELINE, false, try_tcache, true, arena); if (node == NULL) return (NULL); @@ -50,7 +50,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, arena = arena_choose(tsd, arena); if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena, usize, alignment, &is_zeroed)) == NULL) { - idalloct(tsd, node, try_tcache); + idalloctm(tsd, node, try_tcache, true); return (NULL); } @@ -73,6 +73,33 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, return (ret); } +static extent_node_t * +huge_node_locked(const void *ptr) +{ + extent_node_t *node, key; + + /* Extract from tree of huge allocations. */ + key.addr = __DECONST(void *, ptr); + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + malloc_mutex_unlock(&huge_mtx); + + return (node); +} + +static extent_node_t * +huge_node(const void *ptr) +{ + extent_node_t *node; + + malloc_mutex_lock(&huge_mtx); + node = huge_node_locked(ptr); + malloc_mutex_unlock(&huge_mtx); + + return (node); +} + #ifdef JEMALLOC_JET #undef huge_dalloc_junk #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) @@ -102,7 +129,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, { size_t usize_next; bool zeroed; - extent_node_t *node, key; + extent_node_t *node; arena_t *arena; /* Increase usize to incorporate extra. */ @@ -126,10 +153,7 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, zeroed = true; malloc_mutex_lock(&huge_mtx); - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); arena = node->arena; /* Update the size of the huge allocation. */ assert(node->size != usize); @@ -159,7 +183,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) { size_t sdiff; bool zeroed; - extent_node_t *node, key; + extent_node_t *node; arena_t *arena; sdiff = CHUNK_CEILING(usize) - usize; @@ -172,10 +196,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) } malloc_mutex_lock(&huge_mtx); - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); arena = node->arena; /* Update the size of the huge allocation. */ node->size = usize; @@ -190,7 +211,7 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) static bool huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { size_t usize; - extent_node_t *node, key; + extent_node_t *node; arena_t *arena; bool is_zeroed_subchunk, is_zeroed_chunk; @@ -201,10 +222,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { } malloc_mutex_lock(&huge_mtx); - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); arena = node->arena; is_zeroed_subchunk = node->zeroed; malloc_mutex_unlock(&huge_mtx); @@ -342,77 +360,44 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, void huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache) { - extent_node_t *node, key; + extent_node_t *node; malloc_mutex_lock(&huge_mtx); - /* Extract from tree of huge allocations. */ - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); + node = huge_node_locked(ptr); extent_tree_ad_remove(&huge, node); malloc_mutex_unlock(&huge_mtx); huge_dalloc_junk(node->addr, node->size); arena_chunk_dalloc_huge(node->arena, node->addr, node->size); - idalloct(tsd, node, try_tcache); + idalloctm(tsd, node, try_tcache, true); +} + +arena_t * +huge_aalloc(const void *ptr) +{ + + return (huge_node(ptr)->arena); } size_t huge_salloc(const void *ptr) { - size_t ret; - extent_node_t *node, key; - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - ret = node->size; - - malloc_mutex_unlock(&huge_mtx); - - return (ret); + return (huge_node(ptr)->size); } prof_tctx_t * huge_prof_tctx_get(const void *ptr) { - prof_tctx_t *ret; - extent_node_t *node, key; - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - ret = node->prof_tctx; - - malloc_mutex_unlock(&huge_mtx); - - return (ret); + return (huge_node(ptr)->prof_tctx); } void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { - extent_node_t *node, key; - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - node->prof_tctx = tctx; - - malloc_mutex_unlock(&huge_mtx); + huge_node(ptr)->prof_tctx = tctx; } bool diff --git a/src/jemalloc.c b/src/jemalloc.c index 632c8d3e..d1fa674c 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -289,45 +289,34 @@ a0get(void) } static void * -a0imalloc(size_t size, bool zero) +a0ialloc(size_t size, bool zero, bool is_metadata) { - void *ret; if (unlikely(malloc_init_a0())) return (NULL); - if (likely(size <= arena_maxclass)) - ret = arena_malloc(NULL, a0get(), size, zero, false); - else - ret = huge_malloc(NULL, a0get(), size, zero, false); - - return (ret); + return (iallocztm(NULL, size, zero, false, is_metadata, a0get())); } static void -a0idalloc(void *ptr) +a0idalloc(void *ptr, bool is_metadata) { - arena_chunk_t *chunk; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - arena_dalloc(NULL, chunk, ptr, false); - else - huge_dalloc(NULL, ptr, false); + idalloctm(NULL, ptr, false, is_metadata); } void * -a0malloc(size_t size, bool zero) +a0malloc(size_t size) { - return (a0imalloc(size, zero)); + return (a0ialloc(size, false, true)); } void a0dalloc(void *ptr) { - a0idalloc(ptr); + a0idalloc(ptr, true); } /* @@ -343,7 +332,7 @@ bootstrap_malloc(size_t size) if (unlikely(size == 0)) size = 1; - return (a0imalloc(size, false)); + return (a0ialloc(size, false, false)); } void * @@ -357,7 +346,7 @@ bootstrap_calloc(size_t num, size_t size) num_size = 1; } - return (a0imalloc(num_size, true)); + return (a0ialloc(num_size, true, false)); } void @@ -367,7 +356,7 @@ bootstrap_free(void *ptr) if (unlikely(ptr == NULL)) return; - a0idalloc(ptr); + a0idalloc(ptr, false); } /* Create a new arena and insert it into the arenas array at index ind. */ @@ -382,7 +371,7 @@ arena_init_locked(unsigned ind) unsigned narenas_new = narenas_total + 1; arena_t **arenas_new = (arena_t **)a0malloc(CACHELINE_CEILING(narenas_new * - sizeof(arena_t *)), false); + sizeof(arena_t *))); if (arenas_new == NULL) return (NULL); memcpy(arenas_new, arenas, narenas_total * sizeof(arena_t *)); @@ -519,7 +508,7 @@ arena_get_hard(tsd_t *tsd, unsigned ind, bool init_if_missing) if (!*arenas_cache_bypassp) { *arenas_cache_bypassp = true; arenas_cache = (arena_t **)a0malloc(sizeof(arena_t *) * - narenas_cache, false); + narenas_cache); *arenas_cache_bypassp = false; } else arenas_cache = NULL; @@ -1202,6 +1191,8 @@ malloc_init_hard_a0_locked(void) arena_boot(); if (config_tcache && tcache_boot()) return (true); + if (config_tcache && tcache_boot()) + malloc_mutex_unlock(&init_lock); if (huge_boot()) return (true); if (malloc_mutex_init(&arenas_lock)) diff --git a/src/prof.c b/src/prof.c index 1103cc94..06f5499f 100644 --- a/src/prof.c +++ b/src/prof.c @@ -532,8 +532,8 @@ prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) /* * Create a single allocation that has space for vec of length bt->len. */ - prof_gctx_t *gctx = (prof_gctx_t *)imalloc(tsd, offsetof(prof_gctx_t, - vec) + (bt->len * sizeof(void *))); + prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, offsetof(prof_gctx_t, + vec) + (bt->len * sizeof(void *)), false, true, true, NULL); if (gctx == NULL) return (NULL); gctx->lock = prof_gctx_mutex_choose(); @@ -574,7 +574,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, prof_leave(tsd, tdata_self); /* Destroy gctx. */ malloc_mutex_unlock(gctx->lock); - idalloc(tsd, gctx); + idalloctm(tsd, gctx, true, true); } else { /* * Compensate for increment in prof_tctx_destroy() or @@ -674,7 +674,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) prof_tdata_destroy(tsd, tdata, false); if (destroy_tctx) - idalloc(tsd, tctx); + idalloctm(tsd, tctx, true, true); } static bool @@ -703,7 +703,7 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { /* OOM. */ prof_leave(tsd, tdata); - idalloc(tsd, gctx.v); + idalloctm(tsd, gctx.v, true, true); return (true); } new_gctx = true; @@ -760,7 +760,8 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) return (NULL); /* Link a prof_tctx_t into gctx for this thread. */ - ret.v = imalloc(tsd, sizeof(prof_tctx_t)); + ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, true, true, + NULL); if (ret.p == NULL) { if (new_gctx) prof_gctx_try_destroy(tsd, tdata, gctx, tdata); @@ -778,7 +779,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) if (error) { if (new_gctx) prof_gctx_try_destroy(tsd, tdata, gctx, tdata); - idalloc(tsd, ret.v); + idalloctm(tsd, ret.v, true, true); return (NULL); } malloc_mutex_lock(gctx->lock); @@ -1158,7 +1159,7 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) to_destroy); tctx_tree_remove(&gctx->tctxs, to_destroy); - idalloc(tsd, to_destroy); + idalloctm(tsd, to_destroy, true, true); } else next = NULL; } while (next != NULL); @@ -1640,7 +1641,8 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, cassert(config_prof); /* Initialize an empty cache for this thread. */ - tdata = (prof_tdata_t *)imalloc(tsd, sizeof(prof_tdata_t)); + tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t), false, + true, true, NULL); if (tdata == NULL) return (NULL); @@ -1653,7 +1655,7 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) { - idalloc(tsd, tdata); + idalloctm(tsd, tdata, true, true); return (NULL); } @@ -1706,9 +1708,9 @@ prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, tdata_tree_remove(&tdatas, tdata); if (tdata->thread_name != NULL) - idalloc(tsd, tdata->thread_name); + idalloctm(tsd, tdata->thread_name, true, true); ckh_delete(tsd, &tdata->bt2tctx); - idalloc(tsd, tdata); + idalloctm(tsd, tdata, true, true); } static void @@ -1869,7 +1871,7 @@ prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) if (size == 1) return (""); - ret = imalloc(tsd, size); + ret = iallocztm(tsd, size, false, true, true, NULL); if (ret == NULL) return (NULL); memcpy(ret, thread_name, size); @@ -1901,7 +1903,7 @@ prof_thread_name_set(tsd_t *tsd, const char *thread_name) return (EAGAIN); if (tdata->thread_name != NULL) { - idalloc(tsd, tdata->thread_name); + idalloctm(tsd, tdata->thread_name, true, true); tdata->thread_name = NULL; } if (strlen(s) > 0) diff --git a/src/quarantine.c b/src/quarantine.c index 12c37e0a..094b44d3 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -26,8 +26,9 @@ quarantine_init(tsd_t *tsd, size_t lg_maxobjs) assert(tsd_nominal(tsd)); - quarantine = (quarantine_t *)imalloc(tsd, offsetof(quarantine_t, objs) + - ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); + quarantine = (quarantine_t *)iallocztm(tsd, offsetof(quarantine_t, objs) + + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)), false, true, + true, NULL); if (quarantine == NULL) return (NULL); quarantine->curbytes = 0; @@ -54,7 +55,7 @@ quarantine_alloc_hook_work(tsd_t *tsd) if (tsd_quarantine_get(tsd) == NULL) tsd_quarantine_set(tsd, quarantine); else - idalloc(tsd, quarantine); + idalloctm(tsd, quarantine, true, true); } static quarantine_t * @@ -86,7 +87,7 @@ quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * sizeof(quarantine_obj_t)); } - idalloc(tsd, quarantine); + idalloctm(tsd, quarantine, true, true); tsd_quarantine_set(tsd, ret); return (ret); @@ -176,7 +177,7 @@ quarantine_cleanup(tsd_t *tsd) quarantine = tsd_quarantine_get(tsd); if (quarantine != NULL) { quarantine_drain(tsd, quarantine, 0); - idalloc(tsd, quarantine); + idalloctm(tsd, quarantine, true, true); tsd_quarantine_set(tsd, NULL); } } diff --git a/src/stats.c b/src/stats.c index 2b3da645..865f7757 100644 --- a/src/stats.c +++ b/src/stats.c @@ -265,6 +265,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned nthreads; const char *dss; size_t page, pactive, pdirty, mapped; + size_t metadata_mapped, metadata_allocated; uint64_t npurge, nmadvise, purged; size_t small_allocated; uint64_t small_nmalloc, small_ndalloc, small_nrequests; @@ -331,6 +332,12 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); + CTL_I_GET("stats.arenas.0.metadata.mapped", &metadata_mapped, size_t); + CTL_I_GET("stats.arenas.0.metadata.allocated", &metadata_allocated, + size_t); + malloc_cprintf(write_cb, cbopaque, + "metadata: mapped: %zu, allocated: %zu\n", metadata_mapped, + metadata_allocated); if (bins) stats_arena_bins_print(write_cb, cbopaque, i); @@ -539,17 +546,18 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, if (config_stats) { size_t *cactive; - size_t allocated, active, mapped; + size_t allocated, active, metadata, mapped; size_t chunks_current, chunks_high; uint64_t chunks_total; CTL_GET("stats.cactive", &cactive, size_t *); CTL_GET("stats.allocated", &allocated, size_t); CTL_GET("stats.active", &active, size_t); + CTL_GET("stats.metadata", &metadata, size_t); CTL_GET("stats.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, - "Allocated: %zu, active: %zu, mapped: %zu\n", - allocated, active, mapped); + "Allocated: %zu, active: %zu, metadata: %zu, mapped: %zu\n", + allocated, active, metadata, mapped); malloc_cprintf(write_cb, cbopaque, "Current active ceiling: %zu\n", atomic_read_z(cactive)); diff --git a/src/tcache.c b/src/tcache.c index 34224ec4..d638015f 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -298,7 +298,7 @@ tcache_create(tsd_t *tsd, arena_t *arena) /* Avoid false cacheline sharing. */ size = sa2u(size, CACHELINE); - tcache = ipalloct(tsd, size, CACHELINE, true, false, arena); + tcache = ipallocztm(tsd, size, CACHELINE, true, false, true, arena); if (tcache == NULL) return (NULL); @@ -353,7 +353,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) arena_prof_accum(tcache->arena, tcache->prof_accumbytes)) prof_idump(); - idalloct(tsd, tcache, false); + idalloctm(tsd, tcache, false, true); } void diff --git a/src/tsd.c b/src/tsd.c index 00d8f95f..3b59acff 100644 --- a/src/tsd.c +++ b/src/tsd.c @@ -15,7 +15,7 @@ void * malloc_tsd_malloc(size_t size) { - return (a0malloc(CACHELINE_CEILING(size), false)); + return (a0malloc(CACHELINE_CEILING(size))); } void From eee27b2a38d6bb741d9de5e028d5b23e2f4ec4cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Marie?= Date: Sun, 25 Jan 2015 15:12:28 +0100 Subject: [PATCH 200/352] huge_node_locked don't have to unlock huge_mtx in src/huge.c, after each call of huge_node_locked(), huge_mtx is already unlocked. don't unlock it twice (it is a undefined behaviour). --- src/huge.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/huge.c b/src/huge.c index c4d1ebc6..84a1ab23 100644 --- a/src/huge.c +++ b/src/huge.c @@ -83,7 +83,6 @@ huge_node_locked(const void *ptr) node = extent_tree_ad_search(&huge, &key); assert(node != NULL); assert(node->addr == ptr); - malloc_mutex_unlock(&huge_mtx); return (node); } From 77d597ebb23aa47a4a0112c294ad6a68857f450c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Marie?= Date: Sun, 25 Jan 2015 10:18:32 +0100 Subject: [PATCH 201/352] add openbsd support --- configure.ac | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configure.ac b/configure.ac index 0a4f01e8..5e93a5d5 100644 --- a/configure.ac +++ b/configure.ac @@ -283,6 +283,11 @@ case "${host}" in abi="elf" AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) ;; + *-*-openbsd*) + CFLAGS="$CFLAGS" + abi="elf" + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + ;; *-*-linux*) CFLAGS="$CFLAGS" CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" From 0fd663e9c5336089a98e8a2a0cf5419b534f045f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 25 Jan 2015 17:31:24 -0800 Subject: [PATCH 202/352] Avoid pointless chunk_recycle() call. Avoid calling chunk_recycle() for mmap()ed chunks if config_munmap is disabled, in which case there are never any recyclable chunks. This resolves #164. --- src/chunk.c | 52 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/src/chunk.c b/src/chunk.c index b9a24416..6d5f84f5 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -132,6 +132,19 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, return (ret); } +static void * +chunk_alloc_core_dss(void *new_addr, size_t size, size_t alignment, bool base, + bool *zero) +{ + void *ret; + + if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, + new_addr, size, alignment, base, zero)) != NULL) + return (ret); + ret = chunk_alloc_dss(new_addr, size, alignment, zero); + return (ret); +} + /* * If the caller specifies (!*zero), it is still possible to receive zeroed * memory, in which case *zero is toggled to true. arena_chunk_alloc() takes @@ -150,31 +163,26 @@ chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, assert((alignment & chunksize_mask) == 0); /* "primary" dss. */ - if (have_dss && dss_prec == dss_prec_primary) { - if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, - new_addr, size, alignment, base, zero)) != NULL) - return (ret); - if ((ret = chunk_alloc_dss(new_addr, size, alignment, zero)) - != NULL) - return (ret); - } - /* mmap. */ - if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, new_addr, - size, alignment, base, zero)) != NULL) + if (have_dss && dss_prec == dss_prec_primary && (ret = + chunk_alloc_core_dss(new_addr, size, alignment, base, zero)) != + NULL) return (ret); - /* Requesting an address not implemented for chunk_alloc_mmap(). */ - if (new_addr == NULL && - (ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) + /* mmap. */ + if (!config_munmap && (ret = chunk_recycle(&chunks_szad_mmap, + &chunks_ad_mmap, new_addr, size, alignment, base, zero)) != NULL) + return (ret); + /* + * Requesting an address is not implemented for chunk_alloc_mmap(), so + * only call it if (new_addr == NULL). + */ + if (new_addr == NULL && (ret = chunk_alloc_mmap(size, alignment, zero)) + != NULL) return (ret); /* "secondary" dss. */ - if (have_dss && dss_prec == dss_prec_secondary) { - if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, - new_addr, size, alignment, base, zero)) != NULL) - return (ret); - if ((ret = chunk_alloc_dss(new_addr, size, alignment, zero)) - != NULL) - return (ret); - } + if (have_dss && dss_prec == dss_prec_secondary && (ret = + chunk_alloc_core_dss(new_addr, size, alignment, base, zero)) != + NULL) + return (ret); /* All strategies for allocation failed. */ return (NULL); From 41f2e692f664da683ae694b17630f5e186aa454c Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 25 Jan 2015 20:15:13 -0800 Subject: [PATCH 203/352] Fix quoting for CONFIG-related sed expression. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 5e93a5d5..4c202591 100644 --- a/configure.ac +++ b/configure.ac @@ -43,7 +43,7 @@ AC_CACHE_CHECK([whether $1 is compilable], dnl ============================================================================ -CONFIG=`echo ${ac_configure_args} | sed -e "s#\'\([^ ]*\)\'#\1#g"` +CONFIG=`echo ${ac_configure_args} | sed -e 's#'"'"'\([^ ]*\)'"'"'#\1#g'` AC_SUBST([CONFIG]) dnl Library revision. From 5b8ed5b7c91939f64f14fc48be84ed20e3f023f4 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 25 Jan 2015 21:16:57 -0800 Subject: [PATCH 204/352] Implement the prof.gdump mallctl. This feature makes it possible to toggle the gdump feature on/off during program execution, whereas the the opt.prof_dump mallctl value can only be set during program startup. This resolves #72. --- doc/jemalloc.xml.in | 28 +++++++++++---- include/jemalloc/internal/private_symbols.txt | 4 +++ include/jemalloc/internal/prof.h | 18 ++++++++++ src/chunk.c | 3 +- src/ctl.c | 27 +++++++++++++++ src/prof.c | 34 +++++++++++++++++++ test/unit/prof_gdump.c | 29 ++++++++++++++-- 7 files changed, 133 insertions(+), 10 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 08fd4eb3..739b33ac 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1215,13 +1215,11 @@ malloc_conf = "xmalloc:true";]]> r- [] - Trigger a memory profile dump every time the total - virtual memory exceeds the previous maximum. Profiles are dumped to - files named according to the pattern - <prefix>.<pid>.<seq>.u<useq>.heap, - where <prefix> is controlled by the opt.prof_prefix - option. This option is disabled by default. + Set the initial state of prof.gdump, which when + enabled triggers a memory profile dump every time the total virtual + memory exceeds the previous maximum. This option is disabled by + default. @@ -1687,6 +1685,22 @@ malloc_conf = "xmalloc:true";]]> option. + + + prof.gdump + (bool) + rw + [] + + When enabled, trigger a memory profile dump every time + the total virtual memory exceeds the previous maximum. Profiles are + dumped to files named according to the pattern + <prefix>.<pid>.<seq>.u<useq>.heap, + where <prefix> is controlled by the opt.prof_prefix + option. + + prof.reset diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index dfa87551..f3fd8262 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -329,6 +329,10 @@ prof_dump_open prof_free prof_free_sampled_object prof_gdump +prof_gdump_get +prof_gdump_get_unlocked +prof_gdump_set +prof_gdump_val prof_idump prof_interval prof_lookup diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index e0818849..b2db6859 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -239,6 +239,9 @@ extern char opt_prof_prefix[ /* Accessed via prof_active_[gs]et{_unlocked,}(). */ extern bool prof_active; +/* Accessed via prof_gdump_[gs]et{_unlocked,}(). */ +extern bool prof_gdump_val; + /* * Profile dump interval, measured in bytes allocated. Each arena triggers a * profile dump when it reaches this threshold. The effect is that the @@ -285,6 +288,8 @@ bool prof_thread_active_get(void); bool prof_thread_active_set(bool active); bool prof_thread_active_init_get(void); bool prof_thread_active_init_set(bool active_init); +bool prof_gdump_get(void); +bool prof_gdump_set(bool active); void prof_boot0(void); void prof_boot1(void); bool prof_boot2(void); @@ -299,6 +304,7 @@ void prof_sample_threshold_update(prof_tdata_t *tdata); #ifndef JEMALLOC_ENABLE_INLINE bool prof_active_get_unlocked(void); +bool prof_gdump_get_unlocked(void); prof_tdata_t *prof_tdata_get(tsd_t *tsd, bool create); bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool commit, prof_tdata_t **tdata_out); @@ -327,6 +333,18 @@ prof_active_get_unlocked(void) return (prof_active); } +JEMALLOC_ALWAYS_INLINE bool +prof_gdump_get_unlocked(void) +{ + + /* + * No locking is used when reading prof_gdump_val in the fast path, so + * there are no guarantees regarding how long it will take for all + * threads to notice state changes. + */ + return (prof_gdump_val); +} + JEMALLOC_ALWAYS_INLINE prof_tdata_t * prof_tdata_get(tsd_t *tsd, bool create) { diff --git a/src/chunk.c b/src/chunk.c index 6d5f84f5..7bfcdb87 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -213,7 +213,8 @@ chunk_register(void *chunk, size_t size, bool base) } else if (config_prof) gdump = false; malloc_mutex_unlock(&chunks_mtx); - if (config_prof && opt_prof && opt_prof_gdump && gdump) + if (config_prof && opt_prof && prof_gdump_get_unlocked() && + gdump) prof_gdump(); } if (config_valgrind) diff --git a/src/ctl.c b/src/ctl.c index b65af520..63a689a3 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -137,6 +137,7 @@ CTL_PROTO(arenas_extend) CTL_PROTO(prof_thread_active_init) CTL_PROTO(prof_active) CTL_PROTO(prof_dump) +CTL_PROTO(prof_gdump) CTL_PROTO(prof_reset) CTL_PROTO(prof_interval) CTL_PROTO(lg_prof_sample) @@ -347,6 +348,7 @@ static const ctl_named_node_t prof_node[] = { {NAME("thread_active_init"), CTL(prof_thread_active_init)}, {NAME("active"), CTL(prof_active)}, {NAME("dump"), CTL(prof_dump)}, + {NAME("gdump"), CTL(prof_gdump)}, {NAME("reset"), CTL(prof_reset)}, {NAME("interval"), CTL(prof_interval)}, {NAME("lg_sample"), CTL(lg_prof_sample)} @@ -1790,6 +1792,31 @@ label_return: return (ret); } +static int +prof_gdump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + bool oldval; + + if (!config_prof) + return (ENOENT); + + if (newp != NULL) { + if (newlen != sizeof(bool)) { + ret = EINVAL; + goto label_return; + } + oldval = prof_gdump_set(*(bool *)newp); + } else + oldval = prof_gdump_get(); + READ(oldval, bool); + + ret = 0; +label_return: + return (ret); +} + static int prof_reset_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) diff --git a/src/prof.c b/src/prof.c index 06f5499f..04b2591c 100644 --- a/src/prof.c +++ b/src/prof.c @@ -44,6 +44,13 @@ static malloc_mutex_t prof_active_mtx; static bool prof_thread_active_init; static malloc_mutex_t prof_thread_active_init_mtx; +/* + * Initialized as opt_prof_gdump, and accessed via + * prof_gdump_[gs]et{_unlocked,}(). + */ +bool prof_gdump_val; +static malloc_mutex_t prof_gdump_mtx; + uint64_t prof_interval = 0; size_t lg_prof_sample; @@ -1961,6 +1968,29 @@ prof_thread_active_init_set(bool active_init) return (active_init_old); } +bool +prof_gdump_get(void) +{ + bool prof_gdump_current; + + malloc_mutex_lock(&prof_gdump_mtx); + prof_gdump_current = prof_gdump_val; + malloc_mutex_unlock(&prof_gdump_mtx); + return (prof_gdump_current); +} + +bool +prof_gdump_set(bool gdump) +{ + bool prof_gdump_old; + + malloc_mutex_lock(&prof_gdump_mtx); + prof_gdump_old = prof_gdump_val; + prof_gdump_val = gdump; + malloc_mutex_unlock(&prof_gdump_mtx); + return (prof_gdump_old); +} + void prof_boot0(void) { @@ -2013,6 +2043,10 @@ prof_boot2(void) if (malloc_mutex_init(&prof_active_mtx)) return (true); + prof_gdump_val = opt_prof_gdump; + if (malloc_mutex_init(&prof_gdump_mtx)) + return (true); + prof_thread_active_init = opt_prof_thread_active_init; if (malloc_mutex_init(&prof_thread_active_init_mtx)) return (true); diff --git a/test/unit/prof_gdump.c b/test/unit/prof_gdump.c index a00b1054..a0e6ee92 100644 --- a/test/unit/prof_gdump.c +++ b/test/unit/prof_gdump.c @@ -21,8 +21,9 @@ prof_dump_open_intercept(bool propagate_err, const char *filename) TEST_BEGIN(test_gdump) { - bool active; - void *p, *q; + bool active, gdump, gdump_old; + void *p, *q, *r, *s; + size_t sz; test_skip_if(!config_prof); @@ -42,8 +43,32 @@ TEST_BEGIN(test_gdump) assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); + gdump = false; + sz = sizeof(gdump_old); + assert_d_eq(mallctl("prof.gdump", &gdump_old, &sz, &gdump, + sizeof(gdump)), 0, + "Unexpected mallctl failure while disabling prof.gdump"); + assert(gdump_old); + did_prof_dump_open = false; + r = mallocx(chunksize, 0); + assert_ptr_not_null(q, "Unexpected mallocx() failure"); + assert_false(did_prof_dump_open, "Unexpected profile dump"); + + gdump = true; + sz = sizeof(gdump_old); + assert_d_eq(mallctl("prof.gdump", &gdump_old, &sz, &gdump, + sizeof(gdump)), 0, + "Unexpected mallctl failure while enabling prof.gdump"); + assert(!gdump_old); + did_prof_dump_open = false; + s = mallocx(chunksize, 0); + assert_ptr_not_null(q, "Unexpected mallocx() failure"); + assert_true(did_prof_dump_open, "Expected a profile dump"); + dallocx(p, 0); dallocx(q, 0); + dallocx(r, 0); + dallocx(s, 0); } TEST_END From 008267b9f6a0e4d92a78f0e8c0697248020fc8d3 Mon Sep 17 00:00:00 2001 From: Felix Janda Date: Tue, 3 Feb 2015 18:58:02 +0100 Subject: [PATCH 205/352] util.c: strerror_r returns char* only on glibc --- src/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util.c b/src/util.c index bfd86af8..a964d700 100644 --- a/src/util.c +++ b/src/util.c @@ -84,7 +84,7 @@ buferror(int err, char *buf, size_t buflen) FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0, (LPSTR)buf, buflen, NULL); return (0); -#elif defined(_GNU_SOURCE) +#elif defined(__GLIBC__) && defined(_GNU_SOURCE) char *b = strerror_r(err, buf, buflen); if (b != buf) { strncpy(buf, b, buflen); From 6505733012458d8fcd0ae8e1f1acdc9ffe33ff35 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 4 Feb 2015 07:16:55 +0900 Subject: [PATCH 206/352] Make opt.lg_dirty_mult work as documented The documentation for opt.lg_dirty_mult says: Per-arena minimum ratio (log base 2) of active to dirty pages. Some dirty unused pages may be allowed to accumulate, within the limit set by the ratio (or one chunk worth of dirty pages, whichever is greater) (...) The restriction in parentheses currently doesn't happen. This makes jemalloc aggressively madvise(), which in turns increases the amount of page faults significantly. For instance, this resulted in several(!) hundred(!) milliseconds startup regression on Firefox for Android. This may require further tweaking, but starting with actually doing what the documentation says is a good start. --- src/arena.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/arena.c b/src/arena.c index 984b8ad2..a5033bf8 100644 --- a/src/arena.c +++ b/src/arena.c @@ -850,6 +850,7 @@ arena_maybe_purge(arena_t *arena) if (opt_lg_dirty_mult < 0) return; threshold = (arena->nactive >> opt_lg_dirty_mult); + threshold = threshold < chunk_npages ? chunk_npages : threshold; /* * Don't purge unless the number of purgeable pages exceeds the * threshold. @@ -893,6 +894,7 @@ arena_compute_npurge(arena_t *arena, bool all) */ if (!all) { size_t threshold = (arena->nactive >> opt_lg_dirty_mult); + threshold = threshold < chunk_npages ? chunk_npages : threshold; npurge = arena->ndirty - threshold; } else From b0808d5f635592cf7b9c487efbf26f13dc60b223 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 3 Feb 2015 12:39:31 -0800 Subject: [PATCH 207/352] Fix shell test to use = instead of ==. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 4c202591..dc8aa02c 100644 --- a/configure.ac +++ b/configure.ac @@ -998,7 +998,7 @@ fi AC_ARG_WITH([lg_page], [AS_HELP_STRING([--with-lg-page=], [Base 2 log of system page size])], [LG_PAGE="$with_lg_page"], [LG_PAGE="detect"]) -if test "x$LG_PAGE" == "xdetect"; then +if test "x$LG_PAGE" = "xdetect"; then AC_CACHE_CHECK([LG_PAGE], [je_cv_lg_page], AC_RUN_IFELSE([AC_LANG_PROGRAM( From f8723572d8b3418f145fc1d5466cca6b8e2530ef Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 3 Feb 2015 12:39:55 -0800 Subject: [PATCH 208/352] Add missing prototypes for bootstrap_{malloc,calloc,free}(). --- include/jemalloc/internal/jemalloc_internal.h.in | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index a4778550..79a23e5e 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -406,7 +406,9 @@ extern uint8_t const size2index_tab[]; arena_t *a0get(void); void *a0malloc(size_t size); void a0dalloc(void *ptr); -size_t a0allocated(void); +void *bootstrap_malloc(size_t size); +void *bootstrap_calloc(size_t num, size_t size); +void bootstrap_free(void *ptr); arena_t *arenas_extend(unsigned ind); arena_t *arena_init(unsigned ind); unsigned narenas_total_get(void); From 8ddc93293cd8370870f221225ef1e013fbff6d65 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 30 Jan 2015 21:22:54 -0800 Subject: [PATCH 209/352] Fix chunk_recycle()'s new_addr functionality. Fix chunk_recycle()'s new_addr functionality to search by address rather than just size if new_addr is specified. The functionality added by a95018ee819abf897562d9d1f3bc31d4dd725a8d (Attempt to expand huge allocations in-place.) only worked if the two search orders happened to return the same results (e.g. in simple test cases). --- src/chunk.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/chunk.c b/src/chunk.c index 7bfcdb87..a3ae548a 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -48,6 +48,8 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t alloc_size, leadsize, trailsize; bool zeroed; + assert(new_addr == NULL || alignment == chunksize); + if (base) { /* * This function may need to call base_node_{,de}alloc(), but @@ -65,13 +67,15 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, key.addr = new_addr; key.size = alloc_size; malloc_mutex_lock(&chunks_mtx); - node = extent_tree_szad_nsearch(chunks_szad, &key); - if (node == NULL || (new_addr && node->addr != new_addr)) { + node = (new_addr != NULL) ? extent_tree_ad_search(chunks_ad, &key) : + extent_tree_szad_nsearch(chunks_szad, &key); + if (node == NULL) { malloc_mutex_unlock(&chunks_mtx); return (NULL); } leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) - (uintptr_t)node->addr; + assert(new_addr == NULL || leadsize == 0); assert(node->size >= leadsize + size); trailsize = node->size - leadsize - size; ret = (void *)((uintptr_t)node->addr + leadsize); From a55dfa4b0af68f372782e130031483ad73cf7eec Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 2 Feb 2015 13:49:08 -0800 Subject: [PATCH 210/352] Implement more atomic operations. - atomic_*_p(). - atomic_cas_*(). - atomic_write_*(). --- include/jemalloc/internal/atomic.h | 544 ++++++++++++++---- include/jemalloc/internal/private_symbols.txt | 7 + test/unit/atomic.c | 85 ++- 3 files changed, 485 insertions(+), 151 deletions(-) diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h index 23ac93ff..f8bd62ec 100644 --- a/include/jemalloc/internal/atomic.h +++ b/include/jemalloc/internal/atomic.h @@ -11,6 +11,7 @@ #define atomic_read_uint64(p) atomic_add_uint64(p, 0) #define atomic_read_uint32(p) atomic_add_uint32(p, 0) +#define atomic_read_p(p) atomic_add_p(p, NULL) #define atomic_read_z(p) atomic_add_z(p, 0) #define atomic_read_u(p) atomic_add_u(p, 0) @@ -19,88 +20,54 @@ #ifdef JEMALLOC_H_INLINES /* - * All functions return the arithmetic result of the atomic operation. Some - * atomic operation APIs return the value prior to mutation, in which case the - * following functions must redundantly compute the result so that it can be - * returned. These functions are normally inlined, so the extra operations can - * be optimized away if the return values aren't used by the callers. + * All arithmetic functions return the arithmetic result of the atomic + * operation. Some atomic operation APIs return the value prior to mutation, in + * which case the following functions must redundantly compute the result so + * that it can be returned. These functions are normally inlined, so the extra + * operations can be optimized away if the return values aren't used by the + * callers. * + * atomic_read_( *p) { return (*p); } * atomic_add_( *p, x) { return (*p + x); } * atomic_sub_( *p, x) { return (*p - x); } + * bool atomic_cas_( *p, c, s) + * { + * if (*p != c) + * return (true); + * *p = s; + * return (false); + * } + * void atomic_write_( *p, x) { *p = x; } */ #ifndef JEMALLOC_ENABLE_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); +bool atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s); +void atomic_write_uint64(uint64_t *p, uint64_t x); uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); +bool atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s); +void atomic_write_uint32(uint32_t *p, uint32_t x); +void *atomic_add_p(void **p, void *x); +void *atomic_sub_p(void **p, void *x); +bool atomic_cas_p(void **p, void *c, void *s); +void atomic_write_p(void **p, void *x); size_t atomic_add_z(size_t *p, size_t x); size_t atomic_sub_z(size_t *p, size_t x); +bool atomic_cas_z(size_t *p, size_t c, size_t s); +void atomic_write_z(size_t *p, size_t x); unsigned atomic_add_u(unsigned *p, unsigned x); unsigned atomic_sub_u(unsigned *p, unsigned x); +bool atomic_cas_u(unsigned *p, unsigned c, unsigned s); +void atomic_write_u(unsigned *p, unsigned x); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) /******************************************************************************/ /* 64-bit operations. */ #if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) -# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 -JEMALLOC_INLINE uint64_t -atomic_add_uint64(uint64_t *p, uint64_t x) -{ - - return (__sync_add_and_fetch(p, x)); -} - -JEMALLOC_INLINE uint64_t -atomic_sub_uint64(uint64_t *p, uint64_t x) -{ - - return (__sync_sub_and_fetch(p, x)); -} -# elif (defined(_MSC_VER)) -JEMALLOC_INLINE uint64_t -atomic_add_uint64(uint64_t *p, uint64_t x) -{ - - return (InterlockedExchangeAdd64(p, x) + x); -} - -JEMALLOC_INLINE uint64_t -atomic_sub_uint64(uint64_t *p, uint64_t x) -{ - - return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x); -} -# elif (defined(JEMALLOC_C11ATOMICS)) -JEMALLOC_INLINE uint64_t -atomic_add_uint64(uint64_t *p, uint64_t x) -{ - volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; - return (atomic_fetch_add(a, x) + x); -} - -JEMALLOC_INLINE uint64_t -atomic_sub_uint64(uint64_t *p, uint64_t x) -{ - volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; - return (atomic_fetch_sub(a, x) - x); -} -# elif (defined(JEMALLOC_OSATOMIC)) -JEMALLOC_INLINE uint64_t -atomic_add_uint64(uint64_t *p, uint64_t x) -{ - - return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); -} - -JEMALLOC_INLINE uint64_t -atomic_sub_uint64(uint64_t *p, uint64_t x) -{ - - return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); -} -# elif (defined(__amd64__) || defined(__x86_64__)) +# if (defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { @@ -130,6 +97,62 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (t + x); } + +JEMALLOC_INLINE bool +atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) +{ + uint8_t success; + + asm volatile ( + "lock; cmpxchgq %4, %0;" + "sete %1;" + : "=m" (*p), "=a" (success) /* Outputs. */ + : "m" (*p), "a" (c), "r" (s) /* Inputs. */ + : "memory" /* Clobbers. */ + ); + + return (!(bool)success); +} + +JEMALLOC_INLINE void +atomic_write_uint64(uint64_t *p, uint64_t x) +{ + + asm volatile ( + "lock; xchgq %1, %0;" + : "=m" (*p), "+r" (x) /* Outputs. */ + : "m" (*p) /* Inputs. */ + : "memory" /* Clobbers. */ + ); +} +# elif (defined(JEMALLOC_C11ATOMICS)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; + return (atomic_fetch_add(a, x) + x); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + volatile atomic_uint_least64_t *a = (volatile atomic_uint_least64_t *)p; + return (atomic_fetch_sub(a, x) - x); +} + +JEMALLOC_INLINE bool +atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) +{ + + return (!atomic_compare_exchange_strong(p, &c, s)); +} + +JEMALLOC_INLINE void +atomic_write_uint64(uint64_t *p, uint64_t x) +{ + + atomic_store(p, x); +} # elif (defined(JEMALLOC_ATOMIC9)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) @@ -152,7 +175,88 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); } -# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) + +JEMALLOC_INLINE bool +atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) +{ + + assert(sizeof(uint64_t) == sizeof(unsigned long)); + + return (!atomic_cmpset_long(p, (unsigned long)c, (unsigned long)s)); +} + +JEMALLOC_INLINE void +atomic_write_uint64(uint64_t *p, uint64_t x) +{ + + assert(sizeof(uint64_t) == sizeof(unsigned long)); + + atomic_store_rel_long(p, x); +} +# elif (defined(JEMALLOC_OSATOMIC)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); +} + +JEMALLOC_INLINE bool +atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) +{ + + return (!OSAtomicCompareAndSwap64(c, s, (int64_t *)p)); +} + +JEMALLOC_INLINE void +atomic_write_uint64(uint64_t *p, uint64_t x) +{ + uint64_t o; + + /*The documented OSAtomic*() API does not expose an atomic exchange. */ + do { + o = atomic_read_uint64(p); + } while (atomic_cas_uint64(p, o, x)); +} +# elif (defined(_MSC_VER)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (InterlockedExchangeAdd64(p, x) + x); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (InterlockedExchangeAdd64(p, -((int64_t)x)) - x); +} + +JEMALLOC_INLINE bool +atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) +{ + uint64_t o; + + o = InterlockedCompareExchange64(p, s, c); + return (o != c); +} + +JEMALLOC_INLINE void +atomic_write_uint64(uint64_t *p, uint64_t x) +{ + + InterlockedExchange64(p, x); +} +# elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8) || \ + defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { @@ -166,6 +270,20 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (__sync_sub_and_fetch(p, x)); } + +JEMALLOC_INLINE bool +atomic_cas_uint64(uint64_t *p, uint64_t c, uint64_t s) +{ + + return (!__sync_bool_compare_and_swap(p, c, s)); +} + +JEMALLOC_INLINE void +atomic_write_uint64(uint64_t *p, uint64_t x) +{ + + __sync_lock_test_and_set(p, x); +} # else # error "Missing implementation for 64-bit atomic operations" # endif @@ -173,63 +291,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) /******************************************************************************/ /* 32-bit operations. */ -#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 -JEMALLOC_INLINE uint32_t -atomic_add_uint32(uint32_t *p, uint32_t x) -{ - - return (__sync_add_and_fetch(p, x)); -} - -JEMALLOC_INLINE uint32_t -atomic_sub_uint32(uint32_t *p, uint32_t x) -{ - - return (__sync_sub_and_fetch(p, x)); -} -#elif (defined(_MSC_VER)) -JEMALLOC_INLINE uint32_t -atomic_add_uint32(uint32_t *p, uint32_t x) -{ - - return (InterlockedExchangeAdd(p, x) + x); -} - -JEMALLOC_INLINE uint32_t -atomic_sub_uint32(uint32_t *p, uint32_t x) -{ - - return (InterlockedExchangeAdd(p, -((int32_t)x)) - x); -} -# elif (defined(JEMALLOC_C11ATOMICS)) -JEMALLOC_INLINE uint32_t -atomic_add_uint32(uint32_t *p, uint32_t x) -{ - volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; - return (atomic_fetch_add(a, x) + x); -} - -JEMALLOC_INLINE uint32_t -atomic_sub_uint32(uint32_t *p, uint32_t x) -{ - volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; - return (atomic_fetch_sub(a, x) - x); -} -#elif (defined(JEMALLOC_OSATOMIC)) -JEMALLOC_INLINE uint32_t -atomic_add_uint32(uint32_t *p, uint32_t x) -{ - - return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); -} - -JEMALLOC_INLINE uint32_t -atomic_sub_uint32(uint32_t *p, uint32_t x) -{ - - return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); -} -#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) +#if (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) { @@ -259,6 +321,62 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (t + x); } + +JEMALLOC_INLINE bool +atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) +{ + uint8_t success; + + asm volatile ( + "lock; cmpxchgl %4, %0;" + "sete %1;" + : "=m" (*p), "=a" (success) /* Outputs. */ + : "m" (*p), "a" (c), "r" (s) /* Inputs. */ + : "memory" + ); + + return (!(bool)success); +} + +JEMALLOC_INLINE void +atomic_write_uint32(uint32_t *p, uint32_t x) +{ + + asm volatile ( + "lock; xchgl %1, %0;" + : "=m" (*p), "+r" (x) /* Outputs. */ + : "m" (*p) /* Inputs. */ + : "memory" /* Clobbers. */ + ); +} +# elif (defined(JEMALLOC_C11ATOMICS)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; + return (atomic_fetch_add(a, x) + x); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + volatile atomic_uint_least32_t *a = (volatile atomic_uint_least32_t *)p; + return (atomic_fetch_sub(a, x) - x); +} + +JEMALLOC_INLINE bool +atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) +{ + + return (!atomic_compare_exchange_strong(p, &c, s)); +} + +JEMALLOC_INLINE void +atomic_write_uint32(uint32_t *p, uint32_t x) +{ + + atomic_store(p, x); +} #elif (defined(JEMALLOC_ATOMIC9)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) @@ -273,7 +391,84 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); } -#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) + +JEMALLOC_INLINE bool +atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) +{ + + return (!atomic_cmpset_32(p, c, s)); +} + +JEMALLOC_INLINE void +atomic_write_uint32(uint32_t *p, uint32_t x) +{ + + atomic_store_rel_32(p, x); +} +#elif (defined(JEMALLOC_OSATOMIC)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); +} + +JEMALLOC_INLINE bool +atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) +{ + + return (!OSAtomicCompareAndSwap32(c, s, (int32_t *)p)); +} + +JEMALLOC_INLINE void +atomic_write_uint32(uint32_t *p, uint32_t x) +{ + uint32_t o; + + /*The documented OSAtomic*() API does not expose an atomic exchange. */ + do { + o = atomic_read_uint32(p); + } while (atomic_cas_uint32(p, o, x)); +} +#elif (defined(_MSC_VER)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (InterlockedExchangeAdd(p, x) + x); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (InterlockedExchangeAdd(p, -((int32_t)x)) - x); +} + +JEMALLOC_INLINE bool +atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) +{ + uint32_t o; + + o = InterlockedCompareExchange32(p, s, c); + return (o != c); +} + +JEMALLOC_INLINE void +atomic_write_uint32(uint32_t *p, uint32_t x) +{ + + InterlockedExchange(p, x); +} +#elif (defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4) || \ + defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) { @@ -287,10 +482,72 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (__sync_sub_and_fetch(p, x)); } + +JEMALLOC_INLINE bool +atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) +{ + + return (!__sync_bool_compare_and_swap(p, c, s)); +} + +JEMALLOC_INLINE void +atomic_write_uint32(uint32_t *p, uint32_t x) +{ + + __sync_lock_test_and_set(p, x); +} #else # error "Missing implementation for 32-bit atomic operations" #endif +/******************************************************************************/ +/* Pointer operations. */ +JEMALLOC_INLINE void * +atomic_add_p(void **p, void *x) +{ + +#if (LG_SIZEOF_PTR == 3) + return ((void *)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); +#elif (LG_SIZEOF_PTR == 2) + return ((void *)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); +#endif +} + +JEMALLOC_INLINE void * +atomic_sub_p(void **p, void *x) +{ + +#if (LG_SIZEOF_PTR == 3) + return ((void *)atomic_add_uint64((uint64_t *)p, + (uint64_t)-((int64_t)x))); +#elif (LG_SIZEOF_PTR == 2) + return ((void *)atomic_add_uint32((uint32_t *)p, + (uint32_t)-((int32_t)x))); +#endif +} + +JEMALLOC_INLINE bool +atomic_cas_p(void **p, void *c, void *s) +{ + +#if (LG_SIZEOF_PTR == 3) + return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); +#elif (LG_SIZEOF_PTR == 2) + return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); +#endif +} + +JEMALLOC_INLINE void +atomic_write_p(void **p, void *x) +{ + +#if (LG_SIZEOF_PTR == 3) + atomic_write_uint64((uint64_t *)p, (uint64_t)x); +#elif (LG_SIZEOF_PTR == 2) + atomic_write_uint32((uint32_t *)p, (uint32_t)x); +#endif +} + /******************************************************************************/ /* size_t operations. */ JEMALLOC_INLINE size_t @@ -317,6 +574,28 @@ atomic_sub_z(size_t *p, size_t x) #endif } +JEMALLOC_INLINE bool +atomic_cas_z(size_t *p, size_t c, size_t s) +{ + +#if (LG_SIZEOF_PTR == 3) + return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); +#elif (LG_SIZEOF_PTR == 2) + return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); +#endif +} + +JEMALLOC_INLINE void +atomic_write_z(size_t *p, size_t x) +{ + +#if (LG_SIZEOF_PTR == 3) + atomic_write_uint64((uint64_t *)p, (uint64_t)x); +#elif (LG_SIZEOF_PTR == 2) + atomic_write_uint32((uint32_t *)p, (uint32_t)x); +#endif +} + /******************************************************************************/ /* unsigned operations. */ JEMALLOC_INLINE unsigned @@ -342,6 +621,29 @@ atomic_sub_u(unsigned *p, unsigned x) (uint32_t)-((int32_t)x))); #endif } + +JEMALLOC_INLINE bool +atomic_cas_u(unsigned *p, unsigned c, unsigned s) +{ + +#if (LG_SIZEOF_INT == 3) + return (atomic_cas_uint64((uint64_t *)p, (uint64_t)c, (uint64_t)s)); +#elif (LG_SIZEOF_INT == 2) + return (atomic_cas_uint32((uint32_t *)p, (uint32_t)c, (uint32_t)s)); +#endif +} + +JEMALLOC_INLINE void +atomic_write_u(unsigned *p, unsigned x) +{ + +#if (LG_SIZEOF_INT == 3) + atomic_write_uint64((uint64_t *)p, (uint64_t)x); +#elif (LG_SIZEOF_INT == 2) + atomic_write_uint32((uint32_t *)p, (uint32_t)x); +#endif +} + /******************************************************************************/ #endif diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index f3fd8262..ba7ab382 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -85,10 +85,17 @@ arena_stats_merge arena_tcache_fill_small arenas_cache_bypass_cleanup arenas_cache_cleanup +atomic_add_p atomic_add_u atomic_add_uint32 atomic_add_uint64 atomic_add_z +atomic_cas_p +atomic_cas_u +atomic_cas_uint32 +atomic_cas_uint64 +atomic_cas_z +atomic_sub_p atomic_sub_u atomic_sub_uint32 atomic_sub_uint64 diff --git a/test/unit/atomic.c b/test/unit/atomic.c index eb6136c7..a774836a 100644 --- a/test/unit/atomic.c +++ b/test/unit/atomic.c @@ -4,48 +4,64 @@ struct p##_test_s { \ t accum0; \ t x; \ + t s; \ }; \ typedef struct p##_test_s p##_test_t; -#define TEST_BODY(p, t, PRI) do { \ +#define TEST_BODY(p, t, tc, ta, PRI) do { \ const p##_test_t tests[] = { \ - {-1, -1}, \ - {-1, 0}, \ - {-1, 1}, \ + {(t)-1, (t)-1, (t)-2}, \ + {(t)-1, (t) 0, (t)-2}, \ + {(t)-1, (t) 1, (t)-2}, \ \ - { 0, -1}, \ - { 0, 0}, \ - { 0, 1}, \ + {(t) 0, (t)-1, (t)-2}, \ + {(t) 0, (t) 0, (t)-2}, \ + {(t) 0, (t) 1, (t)-2}, \ \ - { 1, -1}, \ - { 1, 0}, \ - { 1, 1}, \ + {(t) 1, (t)-1, (t)-2}, \ + {(t) 1, (t) 0, (t)-2}, \ + {(t) 1, (t) 1, (t)-2}, \ \ - {0, -(1 << 22)}, \ - {0, (1 << 22)}, \ - {(1 << 22), -(1 << 22)}, \ - {(1 << 22), (1 << 22)} \ + {(t)0, (t)-(1 << 22), (t)-2}, \ + {(t)0, (t)(1 << 22), (t)-2}, \ + {(t)(1 << 22), (t)-(1 << 22), (t)-2}, \ + {(t)(1 << 22), (t)(1 << 22), (t)-2} \ }; \ unsigned i; \ \ for (i = 0; i < sizeof(tests)/sizeof(p##_test_t); i++) { \ + bool err; \ t accum = tests[i].accum0; \ - assert_u64_eq(atomic_read_##p(&accum), tests[i].accum0, \ - "i=%u", i); \ - assert_u64_eq(atomic_add_##p(&accum, tests[i].x), \ - tests[i].accum0 + tests[i].x, \ - "i=%u, accum=%#"PRI", x=%#"PRI, \ + assert_##ta##_eq(atomic_read_##p(&accum), \ + tests[i].accum0, \ + "Erroneous read, i=%u", i); \ + \ + assert_##ta##_eq(atomic_add_##p(&accum, tests[i].x), \ + (t)((tc)tests[i].accum0 + (tc)tests[i].x), \ + "i=%u, accum=%"PRI", x=%"PRI, \ i, tests[i].accum0, tests[i].x); \ - assert_u64_eq(atomic_read_##p(&accum), accum, \ - "i=%u", i); \ + assert_##ta##_eq(atomic_read_##p(&accum), accum, \ + "Erroneous add, i=%u", i); \ \ accum = tests[i].accum0; \ - assert_u64_eq(atomic_sub_##p(&accum, tests[i].x), \ - tests[i].accum0 - tests[i].x, \ - "i=%u, accum=%#"PRI", x=%#"PRI, \ + assert_##ta##_eq(atomic_sub_##p(&accum, tests[i].x), \ + (t)((tc)tests[i].accum0 - (tc)tests[i].x), \ + "i=%u, accum=%"PRI", x=%"PRI, \ i, tests[i].accum0, tests[i].x); \ - assert_u64_eq(atomic_read_##p(&accum), accum, \ - "i=%u", i); \ + assert_##ta##_eq(atomic_read_##p(&accum), accum, \ + "Erroneous sub, i=%u", i); \ + \ + accum = tests[i].accum0; \ + err = atomic_cas_##p(&accum, tests[i].x, tests[i].s); \ + assert_b_eq(err, tests[i].accum0 != tests[i].x, \ + "Erroneous cas success/failure result"); \ + assert_##ta##_eq(accum, err ? tests[i].accum0 : \ + tests[i].s, "Erroneous cas effect, i=%u", i); \ + \ + accum = tests[i].accum0; \ + atomic_write_##p(&accum, tests[i].s); \ + assert_##ta##_eq(accum, tests[i].s, \ + "Erroneous write, i=%u", i); \ } \ } while (0) @@ -56,7 +72,7 @@ TEST_BEGIN(test_atomic_uint64) #if !(LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) test_skip("64-bit atomic operations not supported"); #else - TEST_BODY(uint64, uint64_t, PRIx64); + TEST_BODY(uint64, uint64_t, uint64_t, u64, PRIx64); #endif } TEST_END @@ -65,7 +81,15 @@ TEST_STRUCT(uint32, uint32_t) TEST_BEGIN(test_atomic_uint32) { - TEST_BODY(uint32, uint32_t, PRIx32); + TEST_BODY(uint32, uint32_t, uint32_t, u32, "#"PRIx32); +} +TEST_END + +TEST_STRUCT(p, void *) +TEST_BEGIN(test_atomic_p) +{ + + TEST_BODY(p, void *, uintptr_t, ptr, "p"); } TEST_END @@ -73,7 +97,7 @@ TEST_STRUCT(z, size_t) TEST_BEGIN(test_atomic_z) { - TEST_BODY(z, size_t, "zx"); + TEST_BODY(z, size_t, size_t, zu, "#zx"); } TEST_END @@ -81,7 +105,7 @@ TEST_STRUCT(u, unsigned) TEST_BEGIN(test_atomic_u) { - TEST_BODY(u, unsigned, "x"); + TEST_BODY(u, unsigned, unsigned, u, "#x"); } TEST_END @@ -92,6 +116,7 @@ main(void) return (test( test_atomic_uint64, test_atomic_uint32, + test_atomic_p, test_atomic_z, test_atomic_u)); } From 918a1a5b3f09cb456c25be9a2555a8fea6a9bb94 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 30 Jan 2015 21:21:16 -0800 Subject: [PATCH 211/352] Reduce extent_node_t size to fit in one cache line. --- include/jemalloc/internal/extent.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index cbfc20a9..f45940c1 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -15,9 +15,6 @@ struct extent_node_s { /* Linkage for the address-ordered tree. */ rb_node(extent_node_t) link_ad; - /* Profile counters, used for huge objects. */ - prof_tctx_t *prof_tctx; - /* Pointer to the extent that this tree node is responsible for. */ void *addr; @@ -27,8 +24,17 @@ struct extent_node_s { /* Arena from which this extent came, if any. */ arena_t *arena; - /* True if zero-filled; used by chunk recycling code. */ - bool zeroed; + /* + * 'prof_tctx' and 'zeroed' are never needed at the same time, so + * overlay them in order to fit extent_node_t in one cache line. + */ + union { + /* Profile counters, used for huge objects. */ + prof_tctx_t *prof_tctx; + + /* True if zero-filled; used by chunk recycling code. */ + bool zeroed; + }; }; typedef rb_tree(extent_node_t) extent_tree_t; From f500a10b2e94852b867334703ad77467dcfd2ddd Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 30 Jan 2015 21:49:19 -0800 Subject: [PATCH 212/352] Refactor base_alloc() to guarantee demand-zeroed memory. Refactor base_alloc() to guarantee that allocations are carved from demand-zeroed virtual memory. This supports sparse data structures such as multi-page radix tree nodes. Enhance base_alloc() to keep track of fragments which were too small to support previous allocation requests, and try to consume them during subsequent requests. This becomes important when request sizes commonly approach or exceed the chunk size (as could radix tree node allocations). --- include/jemalloc/internal/base.h | 1 - include/jemalloc/internal/private_symbols.txt | 1 - src/base.c | 167 +++++++++++------- src/chunk.c | 17 +- src/mutex.c | 6 +- 5 files changed, 114 insertions(+), 78 deletions(-) diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index 18b7a72d..a0798ee2 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -10,7 +10,6 @@ #ifdef JEMALLOC_H_EXTERNS void *base_alloc(size_t size); -void *base_calloc(size_t number, size_t size); extent_node_t *base_node_alloc(void); void base_node_dalloc(extent_node_t *node); size_t base_allocated_get(void); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index ba7ab382..105e6646 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -103,7 +103,6 @@ atomic_sub_z base_alloc base_allocated_get base_boot -base_calloc base_node_alloc base_node_dalloc base_postfork_child diff --git a/src/base.c b/src/base.c index 22f36139..0d1de7fc 100644 --- a/src/base.c +++ b/src/base.c @@ -5,73 +5,117 @@ /* Data. */ static malloc_mutex_t base_mtx; - -/* - * Current pages that are being used for internal memory allocations. These - * pages are carved up in cacheline-size quanta, so that there is no chance of - * false cache line sharing. - */ -static void *base_pages; -static void *base_next_addr; -static void *base_past_addr; /* Addr immediately past base_pages. */ +static extent_tree_t base_avail_szad; static extent_node_t *base_nodes; - static size_t base_allocated; /******************************************************************************/ -static bool -base_pages_alloc(size_t minsize) +static extent_node_t * +base_node_try_alloc_locked(void) { - size_t csize; + extent_node_t *node; - assert(minsize != 0); - csize = CHUNK_CEILING(minsize); - base_pages = chunk_alloc_base(csize); - if (base_pages == NULL) - return (true); - base_next_addr = base_pages; - base_past_addr = (void *)((uintptr_t)base_pages + csize); - - return (false); + if (base_nodes == NULL) + return (NULL); + node = base_nodes; + base_nodes = *(extent_node_t **)node; + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); + return (node); } +static void +base_node_dalloc_locked(extent_node_t *node) +{ + + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); + *(extent_node_t **)node = base_nodes; + base_nodes = node; +} + +/* base_mtx must be held. */ +static extent_node_t * +base_chunk_alloc(size_t minsize) +{ + extent_node_t *node; + size_t csize, nsize; + void *addr; + + assert(minsize != 0); + node = base_node_try_alloc_locked(); + /* Allocate enough space to also carve a node out if necessary. */ + nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0; + csize = CHUNK_CEILING(minsize + nsize); + addr = chunk_alloc_base(csize); + if (addr == NULL) { + if (node != NULL) + base_node_dalloc_locked(node); + return (NULL); + } + if (node == NULL) { + csize -= nsize; + node = (extent_node_t *)((uintptr_t)addr + csize); + if (config_stats) + base_allocated += nsize; + } + node->addr = addr; + node->size = csize; + return (node); +} + +static void * +base_alloc_locked(size_t size) +{ + void *ret; + size_t csize; + extent_node_t *node; + extent_node_t key; + + /* + * Round size up to nearest multiple of the cacheline size, so that + * there is no chance of false cache line sharing. + */ + csize = CACHELINE_CEILING(size); + + key.addr = NULL; + key.size = csize; + node = extent_tree_szad_nsearch(&base_avail_szad, &key); + if (node != NULL) { + /* Use existing space. */ + extent_tree_szad_remove(&base_avail_szad, node); + } else { + /* Try to allocate more space. */ + node = base_chunk_alloc(csize); + } + if (node == NULL) + return (NULL); + + ret = node->addr; + if (node->size > csize) { + node->addr = (void *)((uintptr_t)ret + csize); + node->size -= csize; + extent_tree_szad_insert(&base_avail_szad, node); + } else + base_node_dalloc_locked(node); + if (config_stats) + base_allocated += csize; + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); + return (ret); +} + +/* + * base_alloc() guarantees demand-zeroed memory, in order to make multi-page + * sparse data structures such as radix tree nodes efficient with respect to + * physical memory usage. + */ void * base_alloc(size_t size) { void *ret; - size_t csize; - - /* Round size up to nearest multiple of the cacheline size. */ - csize = CACHELINE_CEILING(size); malloc_mutex_lock(&base_mtx); - /* Make sure there's enough space for the allocation. */ - if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) { - if (base_pages_alloc(csize)) { - malloc_mutex_unlock(&base_mtx); - return (NULL); - } - } - /* Allocate. */ - ret = base_next_addr; - base_next_addr = (void *)((uintptr_t)base_next_addr + csize); - if (config_stats) - base_allocated += csize; + ret = base_alloc_locked(size); malloc_mutex_unlock(&base_mtx); - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); - - return (ret); -} - -void * -base_calloc(size_t number, size_t size) -{ - void *ret = base_alloc(number * size); - - if (ret != NULL) - memset(ret, 0, number * size); - return (ret); } @@ -81,17 +125,9 @@ base_node_alloc(void) extent_node_t *ret; malloc_mutex_lock(&base_mtx); - if (base_nodes != NULL) { - ret = base_nodes; - base_nodes = *(extent_node_t **)ret; - malloc_mutex_unlock(&base_mtx); - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, - sizeof(extent_node_t)); - } else { - malloc_mutex_unlock(&base_mtx); - ret = (extent_node_t *)base_alloc(sizeof(extent_node_t)); - } - + if ((ret = base_node_try_alloc_locked()) == NULL) + ret = (extent_node_t *)base_alloc_locked(sizeof(extent_node_t)); + malloc_mutex_unlock(&base_mtx); return (ret); } @@ -99,10 +135,8 @@ void base_node_dalloc(extent_node_t *node) { - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); malloc_mutex_lock(&base_mtx); - *(extent_node_t **)node = base_nodes; - base_nodes = node; + base_node_dalloc_locked(node); malloc_mutex_unlock(&base_mtx); } @@ -121,9 +155,10 @@ bool base_boot(void) { - base_nodes = NULL; if (malloc_mutex_init(&base_mtx)) return (true); + extent_tree_szad_new(&base_avail_szad); + base_nodes = NULL; return (false); } diff --git a/src/chunk.c b/src/chunk.c index a3ae548a..01180a71 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -232,15 +232,18 @@ chunk_alloc_base(size_t size) void *ret; bool zero; - zero = false; - ret = chunk_alloc_core(NULL, size, chunksize, true, &zero, - chunk_dss_prec_get()); - if (ret == NULL) - return (NULL); - if (chunk_register(ret, size, true)) { + /* + * Directly call chunk_alloc_mmap() rather than chunk_alloc_core() + * because it's critical that chunk_alloc_base() return untouched + * demand-zeroed virtual memory. + */ + zero = true; + ret = chunk_alloc_mmap(size, chunksize, &zero); + if (ret != NULL && chunk_register(ret, size, true)) { chunk_dalloc_core(ret, size); - return (NULL); + ret = NULL; } + return (ret); } diff --git a/src/mutex.c b/src/mutex.c index 788eca38..d86887ee 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -83,8 +83,8 @@ malloc_mutex_init(malloc_mutex_t *mutex) mutex->postponed_next = postponed_mutexes; postponed_mutexes = mutex; } else { - if (_pthread_mutex_init_calloc_cb(&mutex->lock, base_calloc) != - 0) + if (_pthread_mutex_init_calloc_cb(&mutex->lock, + bootstrap_calloc) != 0) return (true); } #else @@ -140,7 +140,7 @@ mutex_boot(void) postpone_init = false; while (postponed_mutexes != NULL) { if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock, - base_calloc) != 0) + bootstrap_calloc) != 0) return (true); postponed_mutexes = postponed_mutexes->postponed_next; } From c810fcea1fa7983ef5bcabe6556cdc19dde6dd8d Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 4 Feb 2015 16:41:55 -0800 Subject: [PATCH 213/352] Add (x != 0) assertion to lg_floor(x). lg_floor(0) is undefined, but depending on compiler options may not cause a crash. This assertion makes it harder to accidentally abuse lg_floor(). --- include/jemalloc/internal/util.h | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index b2b4ab74..5ad4933d 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -136,14 +136,14 @@ JEMALLOC_ALWAYS_INLINE int jemalloc_ffsl(long bitmap) { - return (JEMALLOC_INTERNAL_FFSL(bitmap)); + return (JEMALLOC_INTERNAL_FFSL(bitmap)); } JEMALLOC_ALWAYS_INLINE int jemalloc_ffs(int bitmap) { - return (JEMALLOC_INTERNAL_FFS(bitmap)); + return (JEMALLOC_INTERNAL_FFS(bitmap)); } /* Compute the smallest power of 2 that is >= x. */ @@ -170,6 +170,8 @@ lg_floor(size_t x) { size_t ret; + assert(x != 0); + asm ("bsr %1, %0" : "=r"(ret) // Outputs. : "r"(x) // Inputs. @@ -180,22 +182,26 @@ lg_floor(size_t x) JEMALLOC_INLINE size_t lg_floor(size_t x) { - unsigned long ret; + unsigned long ret; + + assert(x != 0); #if (LG_SIZEOF_PTR == 3) - _BitScanReverse64(&ret, x); + _BitScanReverse64(&ret, x); #elif (LG_SIZEOF_PTR == 2) - _BitScanReverse(&ret, x); + _BitScanReverse(&ret, x); #else # error "Unsupported type sizes for lg_floor()" #endif - return (ret); + return (ret); } #elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) JEMALLOC_INLINE size_t lg_floor(size_t x) { + assert(x != 0); + #if (LG_SIZEOF_PTR == LG_SIZEOF_INT) return (((8 << LG_SIZEOF_PTR) - 1) - __builtin_clz(x)); #elif (LG_SIZEOF_PTR == LG_SIZEOF_LONG) @@ -209,6 +215,8 @@ JEMALLOC_INLINE size_t lg_floor(size_t x) { + assert(x != 0); + x |= (x >> 1); x |= (x >> 2); x |= (x >> 4); From 8d0e04d42f4750970ac3052a6c76379b60aba5dc Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 30 Jan 2015 22:54:08 -0800 Subject: [PATCH 214/352] Refactor rtree to be lock-free. Recent huge allocation refactoring associates huge allocations with arenas, but it remains necessary to quickly look up huge allocation metadata during reallocation/deallocation. A global radix tree remains a good solution to this problem, but locking would have become the primary bottleneck after (upcoming) migration of chunk management from global to per arena data structures. This lock-free implementation uses double-checked reads to traverse the tree, so that in the steady state, each read or write requires only a single atomic operation. This implementation also assures that no more than two tree levels actually exist, through a combination of careful virtual memory allocation which makes large sparse nodes cheap, and skipping the root node on x64 (possible because the top 16 bits are all 0 in practice). --- include/jemalloc/internal/chunk.h | 2 +- .../jemalloc/internal/jemalloc_internal.h.in | 2 +- include/jemalloc/internal/private_symbols.txt | 15 +- include/jemalloc/internal/rtree.h | 342 +++++++++++------- src/chunk.c | 25 +- src/rtree.c | 150 ++++---- test/unit/rtree.c | 77 ++-- 7 files changed, 384 insertions(+), 229 deletions(-) diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 764b7aca..62ac3e73 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -35,7 +35,7 @@ extern malloc_mutex_t chunks_mtx; /* Chunk statistics. */ extern chunk_stats_t stats_chunks; -extern rtree_t *chunks_rtree; +extern rtree_t chunks_rtree; extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 79a23e5e..280501df 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -955,7 +955,7 @@ ivsalloc(const void *ptr, bool demote) { /* Return 0 if ptr is not within a chunk managed by jemalloc. */ - if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0) + if (rtree_get(&chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0) return (0); return (isalloc(ptr, demote)); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 105e6646..7a78f580 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -369,14 +369,21 @@ quarantine_alloc_hook quarantine_alloc_hook_work quarantine_cleanup register_zone +rtree_child_read +rtree_child_read_hard +rtree_child_tryread rtree_delete rtree_get -rtree_get_locked rtree_new -rtree_postfork_child -rtree_postfork_parent -rtree_prefork +rtree_node_valid rtree_set +rtree_start_level +rtree_subkey +rtree_subtree_read +rtree_subtree_read_hard +rtree_subtree_tryread +rtree_val_read +rtree_val_write s2u s2u_compute s2u_lookup diff --git a/include/jemalloc/internal/rtree.h b/include/jemalloc/internal/rtree.h index bc74769f..e86e17c4 100644 --- a/include/jemalloc/internal/rtree.h +++ b/include/jemalloc/internal/rtree.h @@ -1,170 +1,270 @@ /* * This radix tree implementation is tailored to the singular purpose of - * tracking which chunks are currently owned by jemalloc. This functionality - * is mandatory for OS X, where jemalloc must be able to respond to object - * ownership queries. + * associating metadata with chunks that are currently owned by jemalloc. * ******************************************************************************* */ #ifdef JEMALLOC_H_TYPES +typedef struct rtree_node_elm_s rtree_node_elm_t; +typedef struct rtree_level_s rtree_level_t; typedef struct rtree_s rtree_t; /* - * Size of each radix tree node (must be a power of 2). This impacts tree - * depth. + * RTREE_BITS_PER_LEVEL must be a power of two that is no larger than the + * machine address width. */ -#define RTREE_NODESIZE (1U << 16) +#define LG_RTREE_BITS_PER_LEVEL 4 +#define RTREE_BITS_PER_LEVEL (ZU(1) << LG_RTREE_BITS_PER_LEVEL) +#define RTREE_HEIGHT_MAX \ + ((ZU(1) << (LG_SIZEOF_PTR+3)) / RTREE_BITS_PER_LEVEL) -typedef void *(rtree_alloc_t)(size_t); -typedef void (rtree_dalloc_t)(void *); +/* Used for two-stage lock-free node initialization. */ +#define RTREE_NODE_INITIALIZING ((rtree_node_elm_t *)0x1) + +/* + * The node allocation callback function's argument is the number of contiguous + * rtree_node_elm_t structures to allocate, and the resulting memory must be + * zeroed. + */ +typedef rtree_node_elm_t *(rtree_node_alloc_t)(size_t); +typedef void (rtree_node_dalloc_t)(rtree_node_elm_t *); #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +struct rtree_node_elm_s { + union { + rtree_node_elm_t *child; + void *val; + }; +}; + +struct rtree_level_s { + /* + * A non-NULL subtree points to a subtree rooted along the hypothetical + * path to the leaf node corresponding to key 0. Depending on what keys + * have been used to store to the tree, an arbitrary combination of + * subtree pointers may remain NULL. + * + * Suppose keys comprise 48 bits, and LG_RTREE_BITS_PER_LEVEL is 4. + * This results in a 3-level tree, and the leftmost leaf can be directly + * accessed via subtrees[2], the subtree prefixed by 0x0000 (excluding + * 0x00000000) can be accessed via subtrees[1], and the remainder of the + * tree can be accessed via subtrees[0]. + * + * levels[0] : [ | 0x0001******** | 0x0002******** | ...] + * + * levels[1] : [ | 0x00000001**** | 0x00000002**** | ... ] + * + * levels[2] : [val(0x000000000000) | val(0x000000000001) | ...] + * + * This has practical implications on x64, which currently uses only the + * lower 47 bits of virtual address space in userland, thus leaving + * subtrees[0] unused and avoiding a level of tree traversal. + */ + rtree_node_elm_t *subtree; + /* Number of key bits distinguished by this level. */ + unsigned bits; + /* + * Cumulative number of key bits distinguished by traversing to + * corresponding tree level. + */ + unsigned cumbits; +}; + struct rtree_s { - rtree_alloc_t *alloc; - rtree_dalloc_t *dalloc; - malloc_mutex_t mutex; - void **root; - unsigned height; - unsigned level2bits[1]; /* Dynamically sized. */ + rtree_node_alloc_t *alloc; + rtree_node_dalloc_t *dalloc; + unsigned height; + /* + * Precomputed table used to convert from the number of leading 0 key + * bits to which subtree level to start at. + */ + unsigned start_level[RTREE_HEIGHT_MAX]; + rtree_level_t levels[RTREE_HEIGHT_MAX]; }; #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -rtree_t *rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc); +bool rtree_new(rtree_t *rtree, unsigned bits, rtree_node_alloc_t *alloc, + rtree_node_dalloc_t *dalloc); void rtree_delete(rtree_t *rtree); -void rtree_prefork(rtree_t *rtree); -void rtree_postfork_parent(rtree_t *rtree); -void rtree_postfork_child(rtree_t *rtree); +rtree_node_elm_t *rtree_subtree_read_hard(rtree_t *rtree, + unsigned level); +rtree_node_elm_t *rtree_child_read_hard(rtree_t *rtree, + rtree_node_elm_t *elm, unsigned level); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -#ifdef JEMALLOC_DEBUG -uint8_t rtree_get_locked(rtree_t *rtree, uintptr_t key); -#endif -uint8_t rtree_get(rtree_t *rtree, uintptr_t key); -bool rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val); +unsigned rtree_start_level(rtree_t *rtree, uintptr_t key); +uintptr_t rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level); + +bool rtree_node_valid(rtree_node_elm_t *node); +rtree_node_elm_t *rtree_child_tryread(rtree_node_elm_t *elm); +rtree_node_elm_t *rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, + unsigned level); +void *rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm); +void rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, void *val); +rtree_node_elm_t *rtree_subtree_tryread(rtree_t *rtree, unsigned level); +rtree_node_elm_t *rtree_subtree_read(rtree_t *rtree, unsigned level); + +void *rtree_get(rtree_t *rtree, uintptr_t key); +bool rtree_set(rtree_t *rtree, uintptr_t key, void *val); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_)) -#define RTREE_GET_GENERATE(f) \ -/* The least significant bits of the key are ignored. */ \ -JEMALLOC_INLINE uint8_t \ -f(rtree_t *rtree, uintptr_t key) \ -{ \ - uint8_t ret; \ - uintptr_t subkey; \ - unsigned i, lshift, height, bits; \ - void **node, **child; \ - \ - RTREE_LOCK(&rtree->mutex); \ - for (i = lshift = 0, height = rtree->height, node = rtree->root;\ - i < height - 1; \ - i++, lshift += bits, node = child) { \ - bits = rtree->level2bits[i]; \ - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \ - 3)) - bits); \ - child = (void**)node[subkey]; \ - if (child == NULL) { \ - RTREE_UNLOCK(&rtree->mutex); \ - return (0); \ - } \ - } \ - \ - /* \ - * node is a leaf, so it contains values rather than node \ - * pointers. \ - */ \ - bits = rtree->level2bits[i]; \ - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - \ - bits); \ - { \ - uint8_t *leaf = (uint8_t *)node; \ - ret = leaf[subkey]; \ - } \ - RTREE_UNLOCK(&rtree->mutex); \ - \ - RTREE_GET_VALIDATE \ - return (ret); \ +JEMALLOC_INLINE unsigned +rtree_start_level(rtree_t *rtree, uintptr_t key) +{ + unsigned start_level; + + if (unlikely(key == 0)) + return (rtree->height - 1); + + start_level = rtree->start_level[lg_floor(key) >> + LG_RTREE_BITS_PER_LEVEL]; + assert(start_level < rtree->height); + return (start_level); } -#ifdef JEMALLOC_DEBUG -# define RTREE_LOCK(l) malloc_mutex_lock(l) -# define RTREE_UNLOCK(l) malloc_mutex_unlock(l) -# define RTREE_GET_VALIDATE -RTREE_GET_GENERATE(rtree_get_locked) -# undef RTREE_LOCK -# undef RTREE_UNLOCK -# undef RTREE_GET_VALIDATE -#endif +JEMALLOC_INLINE uintptr_t +rtree_subkey(rtree_t *rtree, uintptr_t key, unsigned level) +{ -#define RTREE_LOCK(l) -#define RTREE_UNLOCK(l) -#ifdef JEMALLOC_DEBUG - /* - * Suppose that it were possible for a jemalloc-allocated chunk to be - * munmap()ped, followed by a different allocator in another thread re-using - * overlapping virtual memory, all without invalidating the cached rtree - * value. The result would be a false positive (the rtree would claim that - * jemalloc owns memory that it had actually discarded). This scenario - * seems impossible, but the following assertion is a prudent sanity check. - */ -# define RTREE_GET_VALIDATE \ - assert(rtree_get_locked(rtree, key) == ret); -#else -# define RTREE_GET_VALIDATE -#endif -RTREE_GET_GENERATE(rtree_get) -#undef RTREE_LOCK -#undef RTREE_UNLOCK -#undef RTREE_GET_VALIDATE + return ((key >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - + rtree->levels[level].cumbits)) & ((ZU(1) << + rtree->levels[level].bits) - 1)); +} JEMALLOC_INLINE bool -rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val) +rtree_node_valid(rtree_node_elm_t *node) +{ + + return ((uintptr_t)node > (uintptr_t)RTREE_NODE_INITIALIZING); +} + +JEMALLOC_INLINE rtree_node_elm_t * +rtree_child_tryread(rtree_node_elm_t *elm) +{ + rtree_node_elm_t *child; + + /* Double-checked read (first read may be stale. */ + child = elm->child; + if (!rtree_node_valid(child)) + child = atomic_read_p((void **)&elm->child); + return (child); +} + +JEMALLOC_INLINE rtree_node_elm_t * +rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level) +{ + rtree_node_elm_t *child; + + child = rtree_child_tryread(elm); + if (unlikely(!rtree_node_valid(child))) + child = rtree_child_read_hard(rtree, elm, level); + return (child); +} + +JEMALLOC_INLINE void * +rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm) +{ + + return (atomic_read_p(&elm->val)); +} + +JEMALLOC_INLINE void +rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, void *val) +{ + + atomic_write_p(&elm->val, val); +} + +JEMALLOC_INLINE rtree_node_elm_t * +rtree_subtree_tryread(rtree_t *rtree, unsigned level) +{ + rtree_node_elm_t *subtree; + + /* Double-checked read (first read may be stale. */ + subtree = rtree->levels[level].subtree; + if (!rtree_node_valid(subtree)) + subtree = atomic_read_p((void **)&rtree->levels[level].subtree); + return (subtree); +} + +JEMALLOC_INLINE rtree_node_elm_t * +rtree_subtree_read(rtree_t *rtree, unsigned level) +{ + rtree_node_elm_t *subtree; + + subtree = rtree_subtree_tryread(rtree, level); + if (unlikely(!rtree_node_valid(subtree))) + subtree = rtree_subtree_read_hard(rtree, level); + return (subtree); +} + +JEMALLOC_INLINE void * +rtree_get(rtree_t *rtree, uintptr_t key) { uintptr_t subkey; - unsigned i, lshift, height, bits; - void **node, **child; + unsigned i, start_level; + rtree_node_elm_t *node, *child; - malloc_mutex_lock(&rtree->mutex); - for (i = lshift = 0, height = rtree->height, node = rtree->root; - i < height - 1; - i++, lshift += bits, node = child) { - bits = rtree->level2bits[i]; - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - - bits); - child = (void**)node[subkey]; - if (child == NULL) { - size_t size = ((i + 1 < height - 1) ? sizeof(void *) - : (sizeof(uint8_t))) << rtree->level2bits[i+1]; - child = (void**)rtree->alloc(size); - if (child == NULL) { - malloc_mutex_unlock(&rtree->mutex); - return (true); - } - memset(child, 0, size); - node[subkey] = child; + start_level = rtree_start_level(rtree, key); + + for (i = start_level, node = rtree_subtree_tryread(rtree, start_level); + /**/; i++, node = child) { + if (unlikely(!rtree_node_valid(node))) + return (NULL); + subkey = rtree_subkey(rtree, key, i); + if (i == rtree->height - 1) { + /* + * node is a leaf, so it contains values rather than + * child pointers. + */ + return (rtree_val_read(rtree, &node[subkey])); } + assert(i < rtree->height - 1); + child = rtree_child_tryread(&node[subkey]); } + not_reached(); +} - /* node is a leaf, so it contains values rather than node pointers. */ - bits = rtree->level2bits[i]; - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits); - { - uint8_t *leaf = (uint8_t *)node; - leaf[subkey] = val; +JEMALLOC_INLINE bool +rtree_set(rtree_t *rtree, uintptr_t key, void *val) +{ + uintptr_t subkey; + unsigned i, start_level; + rtree_node_elm_t *node, *child; + + start_level = rtree_start_level(rtree, key); + + node = rtree_subtree_read(rtree, start_level); + if (node == NULL) + return (true); + for (i = start_level; /**/; i++, node = child) { + subkey = rtree_subkey(rtree, key, i); + if (i == rtree->height - 1) { + /* + * node is a leaf, so it contains values rather than + * child pointers. + */ + rtree_val_write(rtree, &node[subkey], val); + return (false); + } + assert(i < rtree->height - 1); + child = rtree_child_read(rtree, &node[subkey], i); + if (child == NULL) + return (true); } - malloc_mutex_unlock(&rtree->mutex); - - return (false); + not_reached(); } #endif diff --git a/src/chunk.c b/src/chunk.c index 01180a71..9ba0b0cf 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -21,7 +21,7 @@ static extent_tree_t chunks_ad_mmap; static extent_tree_t chunks_szad_dss; static extent_tree_t chunks_ad_dss; -rtree_t *chunks_rtree; +rtree_t chunks_rtree; /* Various chunk-related settings. */ size_t chunksize; @@ -200,7 +200,7 @@ chunk_register(void *chunk, size_t size, bool base) assert(CHUNK_ADDR2BASE(chunk) == chunk); if (config_ivsalloc && !base) { - if (rtree_set(chunks_rtree, (uintptr_t)chunk, 1)) + if (rtree_set(&chunks_rtree, (uintptr_t)chunk, chunk)) return (true); } if (config_stats || config_prof) { @@ -395,7 +395,7 @@ chunk_dalloc_core(void *chunk, size_t size) assert((size & chunksize_mask) == 0); if (config_ivsalloc) - rtree_set(chunks_rtree, (uintptr_t)chunk, 0); + rtree_set(&chunks_rtree, (uintptr_t)chunk, NULL); if (config_stats || config_prof) { malloc_mutex_lock(&chunks_mtx); assert(stats_chunks.curchunks >= (size / chunksize)); @@ -415,6 +415,14 @@ chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) return (false); } +static rtree_node_elm_t * +chunks_rtree_node_alloc(size_t nelms) +{ + + return ((rtree_node_elm_t *)base_alloc(nelms * + sizeof(rtree_node_elm_t))); +} + bool chunk_boot(void) { @@ -436,9 +444,8 @@ chunk_boot(void) extent_tree_szad_new(&chunks_szad_dss); extent_tree_ad_new(&chunks_ad_dss); if (config_ivsalloc) { - chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - - opt_lg_chunk, base_alloc, NULL); - if (chunks_rtree == NULL) + if (rtree_new(&chunks_rtree, (ZU(1) << (LG_SIZEOF_PTR+3)) - + opt_lg_chunk, chunks_rtree_node_alloc, NULL)) return (true); } @@ -450,8 +457,6 @@ chunk_prefork(void) { malloc_mutex_prefork(&chunks_mtx); - if (config_ivsalloc) - rtree_prefork(chunks_rtree); chunk_dss_prefork(); } @@ -460,8 +465,6 @@ chunk_postfork_parent(void) { chunk_dss_postfork_parent(); - if (config_ivsalloc) - rtree_postfork_parent(chunks_rtree); malloc_mutex_postfork_parent(&chunks_mtx); } @@ -470,7 +473,5 @@ chunk_postfork_child(void) { chunk_dss_postfork_child(); - if (config_ivsalloc) - rtree_postfork_child(chunks_rtree); malloc_mutex_postfork_child(&chunks_mtx); } diff --git a/src/rtree.c b/src/rtree.c index 2ff93dbe..47d9084e 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -1,75 +1,74 @@ #define JEMALLOC_RTREE_C_ #include "jemalloc/internal/jemalloc_internal.h" -rtree_t * -rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc) +static unsigned +hmin(unsigned ha, unsigned hb) { - rtree_t *ret; - unsigned bits_per_level, bits_in_leaf, height, i; + + return (ha < hb ? ha : hb); +} + +/* Only the most significant bits of keys passed to rtree_[gs]et() are used. */ +bool +rtree_new(rtree_t *rtree, unsigned bits, rtree_node_alloc_t *alloc, + rtree_node_dalloc_t *dalloc) +{ + unsigned bits_in_leaf, height, i; assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3)); - bits_per_level = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void - *)))) - 1; - bits_in_leaf = jemalloc_ffs(pow2_ceil((RTREE_NODESIZE / - sizeof(uint8_t)))) - 1; + bits_in_leaf = (bits % RTREE_BITS_PER_LEVEL) == 0 ? RTREE_BITS_PER_LEVEL + : (bits % RTREE_BITS_PER_LEVEL); if (bits > bits_in_leaf) { - height = 1 + (bits - bits_in_leaf) / bits_per_level; - if ((height-1) * bits_per_level + bits_in_leaf != bits) + height = 1 + (bits - bits_in_leaf) / RTREE_BITS_PER_LEVEL; + if ((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf != bits) height++; - } else { - height = 1; - } - assert((height-1) * bits_per_level + bits_in_leaf >= bits); - - ret = (rtree_t*)alloc(offsetof(rtree_t, level2bits) + - (sizeof(unsigned) * height)); - if (ret == NULL) - return (NULL); - memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) * - height)); - - ret->alloc = alloc; - ret->dalloc = dalloc; - if (malloc_mutex_init(&ret->mutex)) { - if (dalloc != NULL) - dalloc(ret); - return (NULL); - } - ret->height = height; - if (height > 1) { - if ((height-1) * bits_per_level + bits_in_leaf > bits) { - ret->level2bits[0] = (bits - bits_in_leaf) % - bits_per_level; - } else - ret->level2bits[0] = bits_per_level; - for (i = 1; i < height-1; i++) - ret->level2bits[i] = bits_per_level; - ret->level2bits[height-1] = bits_in_leaf; } else - ret->level2bits[0] = bits; + height = 1; + assert((height-1) * RTREE_BITS_PER_LEVEL + bits_in_leaf == bits); - ret->root = (void**)alloc(sizeof(void *) << ret->level2bits[0]); - if (ret->root == NULL) { - if (dalloc != NULL) - dalloc(ret); - return (NULL); + rtree->alloc = alloc; + rtree->dalloc = dalloc; + rtree->height = height; + + /* Root level. */ + rtree->levels[0].subtree = NULL; + rtree->levels[0].bits = (height > 1) ? RTREE_BITS_PER_LEVEL : + bits_in_leaf; + rtree->levels[0].cumbits = rtree->levels[0].bits; + /* Interior levels. */ + for (i = 1; i < height-1; i++) { + rtree->levels[i].subtree = NULL; + rtree->levels[i].bits = RTREE_BITS_PER_LEVEL; + rtree->levels[i].cumbits = rtree->levels[i-1].cumbits + + RTREE_BITS_PER_LEVEL; + } + /* Leaf level. */ + if (height > 1) { + rtree->levels[height-1].subtree = NULL; + rtree->levels[height-1].bits = bits_in_leaf; + rtree->levels[height-1].cumbits = bits; } - memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]); - return (ret); + /* Compute lookup table to be used by rtree_start_level(). */ + for (i = 0; i < RTREE_HEIGHT_MAX; i++) { + rtree->start_level[i] = hmin(RTREE_HEIGHT_MAX - 1 - i, height - + 1); + } + + return (false); } static void -rtree_delete_subtree(rtree_t *rtree, void **node, unsigned level) +rtree_delete_subtree(rtree_t *rtree, rtree_node_elm_t *node, unsigned level) { if (level < rtree->height - 1) { size_t nchildren, i; - nchildren = ZU(1) << rtree->level2bits[level]; + nchildren = ZU(1) << rtree->levels[level].bits; for (i = 0; i < nchildren; i++) { - void **child = (void **)node[i]; + rtree_node_elm_t *child = node[i].child; if (child != NULL) rtree_delete_subtree(rtree, child, level + 1); } @@ -80,28 +79,49 @@ rtree_delete_subtree(rtree_t *rtree, void **node, unsigned level) void rtree_delete(rtree_t *rtree) { + unsigned i; - rtree_delete_subtree(rtree, rtree->root, 0); - rtree->dalloc(rtree); + for (i = 0; i < rtree->height; i++) { + rtree_node_elm_t *subtree = rtree->levels[i].subtree; + if (subtree != NULL) + rtree_delete_subtree(rtree, subtree, i); + } } -void -rtree_prefork(rtree_t *rtree) +static rtree_node_elm_t * +rtree_node_init(rtree_t *rtree, unsigned level, rtree_node_elm_t **elmp) +{ + rtree_node_elm_t *node; + + if (atomic_cas_p((void **)elmp, NULL, RTREE_NODE_INITIALIZING)) { + /* + * Another thread is already in the process of initializing. + * Spin-wait until initialization is complete. + */ + do { + CPU_SPINWAIT; + node = atomic_read_p((void **)elmp); + } while (node == RTREE_NODE_INITIALIZING); + } else { + node = rtree->alloc(ZU(1) << rtree->levels[level].bits); + if (node == NULL) + return (NULL); + atomic_write_p((void **)elmp, node); + } + + return (node); +} + +rtree_node_elm_t * +rtree_subtree_read_hard(rtree_t *rtree, unsigned level) { - malloc_mutex_prefork(&rtree->mutex); + return (rtree_node_init(rtree, level, &rtree->levels[level].subtree)); } -void -rtree_postfork_parent(rtree_t *rtree) +rtree_node_elm_t * +rtree_child_read_hard(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level) { - malloc_mutex_postfork_parent(&rtree->mutex); -} - -void -rtree_postfork_child(rtree_t *rtree) -{ - - malloc_mutex_postfork_child(&rtree->mutex); + return (rtree_node_init(rtree, level, &elm->child)); } diff --git a/test/unit/rtree.c b/test/unit/rtree.c index 77a947d6..556c4a87 100644 --- a/test/unit/rtree.c +++ b/test/unit/rtree.c @@ -1,14 +1,30 @@ #include "test/jemalloc_test.h" +static rtree_node_elm_t * +node_alloc(size_t nelms) +{ + + return (calloc(nelms, sizeof(rtree_node_elm_t))); +} + +static void +node_dalloc(rtree_node_elm_t *node) +{ + + free(node); +} + TEST_BEGIN(test_rtree_get_empty) { unsigned i; for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { - rtree_t *rtree = rtree_new(i, malloc, free); - assert_u_eq(rtree_get(rtree, 0), 0, + rtree_t rtree; + assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc), + "Unexpected rtree_new() failure"); + assert_ptr_eq(rtree_get(&rtree, 0), NULL, "rtree_get() should return NULL for empty tree"); - rtree_delete(rtree); + rtree_delete(&rtree); } } TEST_END @@ -16,19 +32,22 @@ TEST_END TEST_BEGIN(test_rtree_extrema) { unsigned i; + extent_node_t node_a, node_b; for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { - rtree_t *rtree = rtree_new(i, malloc, free); + rtree_t rtree; + assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc), + "Unexpected rtree_new() failure"); - rtree_set(rtree, 0, 1); - assert_u_eq(rtree_get(rtree, 0), 1, + rtree_set(&rtree, 0, &node_a); + assert_ptr_eq(rtree_get(&rtree, 0), &node_a, "rtree_get() should return previously set value"); - rtree_set(rtree, ~((uintptr_t)0), 1); - assert_u_eq(rtree_get(rtree, ~((uintptr_t)0)), 1, + rtree_set(&rtree, ~((uintptr_t)0), &node_b); + assert_ptr_eq(rtree_get(&rtree, ~((uintptr_t)0)), &node_b, "rtree_get() should return previously set value"); - rtree_delete(rtree); + rtree_delete(&rtree); } } TEST_END @@ -40,26 +59,30 @@ TEST_BEGIN(test_rtree_bits) for (i = 1; i < (sizeof(uintptr_t) << 3); i++) { uintptr_t keys[] = {0, 1, (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1}; - rtree_t *rtree = rtree_new(i, malloc, free); + extent_node_t node; + rtree_t rtree; + + assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc), + "Unexpected rtree_new() failure"); for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { - rtree_set(rtree, keys[j], 1); + rtree_set(&rtree, keys[j], &node); for (k = 0; k < sizeof(keys)/sizeof(uintptr_t); k++) { - assert_u_eq(rtree_get(rtree, keys[k]), 1, + assert_ptr_eq(rtree_get(&rtree, keys[k]), &node, "rtree_get() should return previously set " "value and ignore insignificant key bits; " "i=%u, j=%u, k=%u, set key=%#"PRIxPTR", " "get key=%#"PRIxPTR, i, j, k, keys[j], keys[k]); } - assert_u_eq(rtree_get(rtree, - (((uintptr_t)1) << (sizeof(uintptr_t)*8-i))), 0, + assert_ptr_eq(rtree_get(&rtree, + (((uintptr_t)1) << (sizeof(uintptr_t)*8-i))), NULL, "Only leftmost rtree leaf should be set; " "i=%u, j=%u", i, j); - rtree_set(rtree, keys[j], 0); + rtree_set(&rtree, keys[j], NULL); } - rtree_delete(rtree); + rtree_delete(&rtree); } } TEST_END @@ -68,37 +91,41 @@ TEST_BEGIN(test_rtree_random) { unsigned i; sfmt_t *sfmt; -#define NSET 100 +#define NSET 16 #define SEED 42 sfmt = init_gen_rand(SEED); for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { - rtree_t *rtree = rtree_new(i, malloc, free); uintptr_t keys[NSET]; + extent_node_t node; unsigned j; + rtree_t rtree; + + assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc), + "Unexpected rtree_new() failure"); for (j = 0; j < NSET; j++) { keys[j] = (uintptr_t)gen_rand64(sfmt); - rtree_set(rtree, keys[j], 1); - assert_u_eq(rtree_get(rtree, keys[j]), 1, + rtree_set(&rtree, keys[j], &node); + assert_ptr_eq(rtree_get(&rtree, keys[j]), &node, "rtree_get() should return previously set value"); } for (j = 0; j < NSET; j++) { - assert_u_eq(rtree_get(rtree, keys[j]), 1, + assert_ptr_eq(rtree_get(&rtree, keys[j]), &node, "rtree_get() should return previously set value"); } for (j = 0; j < NSET; j++) { - rtree_set(rtree, keys[j], 0); - assert_u_eq(rtree_get(rtree, keys[j]), 0, + rtree_set(&rtree, keys[j], NULL); + assert_ptr_eq(rtree_get(&rtree, keys[j]), NULL, "rtree_get() should return previously set value"); } for (j = 0; j < NSET; j++) { - assert_u_eq(rtree_get(rtree, keys[j]), 0, + assert_ptr_eq(rtree_get(&rtree, keys[j]), NULL, "rtree_get() should return previously set value"); } - rtree_delete(rtree); + rtree_delete(&rtree); } fini_gen_rand(sfmt); #undef NSET From 23694b07457f3aaf9605a4ff6b386f3c897eb624 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 9 Feb 2015 16:19:27 -0800 Subject: [PATCH 215/352] Fix arena_get() for (!init_if_missing && refresh_if_missing) case. Fix arena_get() to refresh the cache as needed in the (!init_if_missing && refresh_if_missing) case. This flaw was introduced by the initial arena_get() implementation, which was part of 8bb3198f72fc7587dc93527f9f19fb5be52fa553 (Refactor/fix arenas manipulation.). --- include/jemalloc/internal/jemalloc_internal.h.in | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 280501df..2b167420 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -755,10 +755,7 @@ arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, arena = arenas_cache[ind]; if (likely(arena != NULL) || !refresh_if_missing) return (arena); - if (init_if_missing) - return (arena_get_hard(tsd, ind, init_if_missing)); - else - return (NULL); + return (arena_get_hard(tsd, ind, init_if_missing)); } #endif From 1cb181ed632e7573fb4eab194e4d216867222d27 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 29 Jan 2015 15:30:47 -0800 Subject: [PATCH 216/352] Implement explicit tcache support. Add the MALLOCX_TCACHE() and MALLOCX_TCACHE_NONE macros, which can be used in conjunction with the *allocx() API. Add the tcache.create, tcache.flush, and tcache.destroy mallctls. This resolves #145. --- doc/jemalloc.xml.in | 106 ++++++++--- include/jemalloc/internal/arena.h | 51 +++--- include/jemalloc/internal/huge.h | 8 +- .../jemalloc/internal/jemalloc_internal.h.in | 134 +++++++------- include/jemalloc/internal/private_symbols.txt | 5 + include/jemalloc/internal/tcache.h | 102 +++++++---- include/jemalloc/jemalloc_macros.h.in | 12 +- src/arena.c | 24 +-- src/ckh.c | 7 +- src/ctl.c | 113 +++++++++++- src/huge.c | 36 ++-- src/jemalloc.c | 158 ++++++++--------- src/prof.c | 35 ++-- src/quarantine.c | 10 +- src/tcache.c | 166 +++++++++++++----- test/unit/mallctl.c | 110 ++++++++++++ 16 files changed, 740 insertions(+), 337 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 739b33ac..da800ded 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -242,7 +242,7 @@ relevant. Use bitwise or (|) operations to specify one or more of the following: - + MALLOCX_LG_ALIGN(la) @@ -252,7 +252,7 @@ that la is within the valid range. - + MALLOCX_ALIGN(a) @@ -262,7 +262,7 @@ validate that a is a power of 2. - + MALLOCX_ZERO Initialize newly allocated memory to contain zero @@ -271,16 +271,38 @@ that are initialized to contain zero bytes. If this macro is absent, newly allocated memory is uninitialized. - + + MALLOCX_TCACHE(tc) + + + Use the thread-specific cache (tcache) specified by + the identifier tc, which must have been + acquired via the tcache.create + mallctl. This macro does not validate that + tc specifies a valid + identifier. + + + MALLOCX_TCACHE_NONE + + Do not use a thread-specific cache (tcache). Unless + MALLOCX_TCACHE(tc) or + MALLOCX_TCACHE_NONE is specified, an + automatically managed tcache will be used under many circumstances. + This macro cannot be used in the same flags + argument as + MALLOCX_TCACHE(tc). + + MALLOCX_ARENA(a) Use the arena specified by the index - a (and by necessity bypass the thread - cache). This macro has no effect for regions that were allocated - via an arena other than the one specified. This macro does not - validate that a specifies an arena index in - the valid range. + a. This macro has no effect for regions that + were allocated via an arena other than the one specified. This + macro does not validate that a specifies an + arena index in the valid range. @@ -1060,12 +1082,11 @@ malloc_conf = "xmalloc:true";]]> r- [] - Thread-specific caching enabled/disabled. When there - are multiple threads, each thread uses a thread-specific cache for - objects up to a certain size. Thread-specific caching allows many - allocations to be satisfied without performing any thread - synchronization, at the cost of increased memory use. See the - Thread-specific caching (tcache) enabled/disabled. When + there are multiple threads, each thread uses a tcache for objects up to + a certain size. Thread-specific caching allows many allocations to be + satisfied without performing any thread synchronization, at the cost of + increased memory use. See the opt.lg_tcache_max option for related tuning information. This option is enabled by default unless running inside [] Maximum size class (log base 2) to cache in the - thread-specific cache. At a minimum, all small size classes are - cached, and at a maximum all large size classes are cached. The + thread-specific cache (tcache). At a minimum, all small size classes + are cached, and at a maximum all large size classes are cached. The default maximum is 32 KiB (2^15). @@ -1339,7 +1360,7 @@ malloc_conf = "xmalloc:true";]]> Enable/disable calling thread's tcache. The tcache is implicitly flushed as a side effect of becoming disabled (see thread.tcache.flush). + linkend="thread.tcache.flush">thread.tcache.flush). @@ -1350,9 +1371,9 @@ malloc_conf = "xmalloc:true";]]> -- [] - Flush calling thread's tcache. This interface releases - all cached objects and internal data structures associated with the - calling thread's thread-specific cache. Ordinarily, this interface + Flush calling thread's thread-specific cache (tcache). + This interface releases all cached objects and internal data structures + associated with the calling thread's tcache. Ordinarily, this interface need not be called, since automatic periodic incremental garbage collection occurs, and the thread cache is automatically discarded when a thread exits. However, garbage collection is triggered by allocation @@ -1399,6 +1420,49 @@ malloc_conf = "xmalloc:true";]]> default. + + + tcache.create + (unsigned) + r- + [] + + Create an explicit thread-specific cache (tcache) and + return an identifier that can be passed to the MALLOCX_TCACHE(tc) + macro to explicitly use the specified cache rather than the + automatically managed one that is used by default. Each explicit cache + can be used by only one thread at a time; the application must assure + that this constraint holds. + + + + + + tcache.flush + (unsigned) + -w + [] + + Flush the specified thread-specific cache (tcache). The + same considerations apply to this interface as to thread.tcache.flush, + except that the tcache will never be automatically be discarded. + + + + + + tcache.destroy + (unsigned) + -w + [] + + Flush the specified thread-specific cache (tcache) and + make the identifier available for use during a future tcache creation. + + + arena.<i>.purge diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 46367f68..5476899d 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -272,7 +272,8 @@ struct arena_s { arena_stats_t stats; /* * List of tcaches for extant threads associated with this arena. - * Stats from these are merged incrementally, and at exit. + * Stats from these are merged incrementally, and at exit if + * opt_stats_print is enabled. */ ql_head(tcache_t) tcache_ql; @@ -387,8 +388,7 @@ extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t extra, size_t alignment, bool zero, - bool try_tcache_alloc, bool try_tcache_dalloc); + size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, @@ -450,13 +450,13 @@ unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, prof_tctx_t *arena_prof_tctx_get(const void *ptr); void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, - bool try_tcache); + tcache_t *tcache); arena_t *arena_aalloc(const void *ptr); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, - bool try_tcache); + tcache_t *tcache); void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, - bool try_tcache); + tcache_t *tcache); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) @@ -943,17 +943,15 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, - bool try_tcache) + tcache_t *tcache) { - tcache_t *tcache; assert(size != 0); assert(size <= arena_maxclass); if (likely(size <= SMALL_MAXCLASS)) { - if (likely(try_tcache) && likely((tcache = tcache_get(tsd, - true)) != NULL)) - return (tcache_alloc_small(tcache, size, zero)); + if (likely(tcache != NULL)) + return (tcache_alloc_small(tsd, tcache, size, zero)); else { arena = arena_choose(tsd, arena); if (unlikely(arena == NULL)) @@ -965,9 +963,8 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, * Initialize tcache after checking size in order to avoid * infinite recursion during tcache initialization. */ - if (try_tcache && size <= tcache_maxclass && likely((tcache = - tcache_get(tsd, true)) != NULL)) - return (tcache_alloc_large(tcache, size, zero)); + if (likely(tcache != NULL) && size <= tcache_maxclass) + return (tcache_alloc_large(tsd, tcache, size, zero)); else { arena = arena_choose(tsd, arena); if (unlikely(arena == NULL)) @@ -1027,10 +1024,9 @@ arena_salloc(const void *ptr, bool demote) } JEMALLOC_ALWAYS_INLINE void -arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) +arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, tcache_t *tcache) { size_t pageind, mapbits; - tcache_t *tcache; assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); @@ -1040,11 +1036,10 @@ arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) assert(arena_mapbits_allocated_get(chunk, pageind) != 0); if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { /* Small allocation. */ - if (likely(try_tcache) && likely((tcache = tcache_get(tsd, - false)) != NULL)) { + if (likely(tcache != NULL)) { index_t binind = arena_ptr_small_binind_get(ptr, mapbits); - tcache_dalloc_small(tcache, ptr, binind); + tcache_dalloc_small(tsd, tcache, ptr, binind); } else arena_dalloc_small(chunk->arena, chunk, ptr, pageind); } else { @@ -1052,9 +1047,8 @@ arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) assert(((uintptr_t)ptr & PAGE_MASK) == 0); - if (try_tcache && size <= tcache_maxclass && likely((tcache = - tcache_get(tsd, false)) != NULL)) - tcache_dalloc_large(tcache, ptr, size); + if (likely(tcache != NULL) && size <= tcache_maxclass) + tcache_dalloc_large(tsd, tcache, ptr, size); else arena_dalloc_large(chunk->arena, chunk, ptr); } @@ -1062,9 +1056,8 @@ arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, bool try_tcache) JEMALLOC_ALWAYS_INLINE void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, - bool try_tcache) + tcache_t *tcache) { - tcache_t *tcache; assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); @@ -1082,10 +1075,9 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, if (likely(size <= SMALL_MAXCLASS)) { /* Small allocation. */ - if (likely(try_tcache) && likely((tcache = tcache_get(tsd, - false)) != NULL)) { + if (likely(tcache != NULL)) { index_t binind = size2index(size); - tcache_dalloc_small(tcache, ptr, binind); + tcache_dalloc_small(tsd, tcache, ptr, binind); } else { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; @@ -1094,9 +1086,8 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, } else { assert(((uintptr_t)ptr & PAGE_MASK) == 0); - if (try_tcache && size <= tcache_maxclass && (tcache = - tcache_get(tsd, false)) != NULL) - tcache_dalloc_large(tcache, ptr, size); + if (likely(tcache != NULL) && size <= tcache_maxclass) + tcache_dalloc_large(tsd, tcache, ptr, size); else arena_dalloc_large(chunk->arena, chunk, ptr); } diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index decb0249..231cc368 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -10,19 +10,19 @@ #ifdef JEMALLOC_H_EXTERNS void *huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, - bool try_tcache); + tcache_t *tcache); void *huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, - bool zero, bool try_tcache); + bool zero, tcache_t *tcache); bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); void *huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, - bool try_tcache_alloc, bool try_tcache_dalloc); + tcache_t *tcache); #ifdef JEMALLOC_JET typedef void (huge_dalloc_junk_t)(void *, size_t); extern huge_dalloc_junk_t *huge_dalloc_junk; #endif -void huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache); +void huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache); arena_t *huge_aalloc(const void *ptr); size_t huge_salloc(const void *ptr); prof_tctx_t *huge_prof_tctx_get(const void *ptr); diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 2b167420..b8c994cb 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -172,7 +172,21 @@ static const bool config_ivsalloc = /* Size class index type. */ typedef unsigned index_t; -#define MALLOCX_ARENA_MASK ((int)~0xff) +/* + * Flags bits: + * + * a: arena + * t: tcache + * 0: unused + * z: zero + * n: alignment + * + * aaaaaaaa aaaatttt tttttttt 0znnnnnn + */ +#define MALLOCX_ARENA_MASK ((int)~0xfffff) +#define MALLOCX_ARENA_MAX 0xffe +#define MALLOCX_TCACHE_MASK ((int)~0xfff000ffU) +#define MALLOCX_TCACHE_MAX 0xffd #define MALLOCX_LG_ALIGN_MASK ((int)0x3f) /* Use MALLOCX_ALIGN_GET() if alignment may not be specified in flags. */ #define MALLOCX_ALIGN_GET_SPECIFIED(flags) \ @@ -181,8 +195,11 @@ typedef unsigned index_t; (MALLOCX_ALIGN_GET_SPECIFIED(flags) & (SIZE_T_MAX-1)) #define MALLOCX_ZERO_GET(flags) \ ((bool)(flags & MALLOCX_ZERO)) + +#define MALLOCX_TCACHE_GET(flags) \ + (((unsigned)((flags & MALLOCX_TCACHE_MASK) >> 8)) - 2) #define MALLOCX_ARENA_GET(flags) \ - (((unsigned)(flags >> 8)) - 1) + (((unsigned)(((unsigned)flags) >> 20)) - 1) /* Smallest size class to support. */ #define TINY_MIN (1U << LG_TINY_MIN) @@ -749,7 +766,7 @@ arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, * ind is invalid, cache is old (too small), or arena to be * initialized. */ - return (refresh_if_missing ? arena_get_hard(tsd, ind, + return (refresh_if_missing ? arena_get_hard(tsd, ind, init_if_missing) : NULL); } arena = arenas_cache[ind]; @@ -778,32 +795,31 @@ arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, #ifndef JEMALLOC_ENABLE_INLINE arena_t *iaalloc(const void *ptr); size_t isalloc(const void *ptr, bool demote); -void *iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, +void *iallocztm(tsd_t *tsd, size_t size, bool zero, tcache_t *tcache, bool is_metadata, arena_t *arena); -void *imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); +void *imalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena); void *imalloc(tsd_t *tsd, size_t size); -void *icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena); +void *icalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena); void *icalloc(tsd_t *tsd, size_t size); void *ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, - bool try_tcache, bool is_metadata, arena_t *arena); + tcache_t *tcache, bool is_metadata, arena_t *arena); void *ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, - bool try_tcache, arena_t *arena); + tcache_t *tcache, arena_t *arena); void *ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero); size_t ivsalloc(const void *ptr, bool demote); size_t u2rz(size_t usize); size_t p2rz(const void *ptr); -void idalloctm(tsd_t *tsd, void *ptr, bool try_tcache, bool is_metadata); -void idalloct(tsd_t *tsd, void *ptr, bool try_tcache); +void idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata); +void idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache); void idalloc(tsd_t *tsd, void *ptr); -void iqalloc(tsd_t *tsd, void *ptr, bool try_tcache); -void isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); -void isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache); +void iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache); +void isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); +void isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); void *iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, - bool try_tcache_dalloc, arena_t *arena); -void *iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, - size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); +void *iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, + size_t alignment, bool zero, tcache_t *tcache, arena_t *arena); void *iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero); bool ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, @@ -853,7 +869,7 @@ isalloc(const void *ptr, bool demote) } JEMALLOC_ALWAYS_INLINE void * -iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, bool is_metadata, +iallocztm(tsd_t *tsd, size_t size, bool zero, tcache_t *tcache, bool is_metadata, arena_t *arena) { void *ret; @@ -861,9 +877,9 @@ iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, bool is_metadata, assert(size != 0); if (likely(size <= arena_maxclass)) - ret = arena_malloc(tsd, arena, size, zero, try_tcache); + ret = arena_malloc(tsd, arena, size, zero, tcache); else - ret = huge_malloc(tsd, arena, size, zero, try_tcache); + ret = huge_malloc(tsd, arena, size, zero, tcache); if (config_stats && is_metadata && likely(ret != NULL)) { arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, config_prof)); @@ -872,36 +888,36 @@ iallocztm(tsd_t *tsd, size_t size, bool zero, bool try_tcache, bool is_metadata, } JEMALLOC_ALWAYS_INLINE void * -imalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) +imalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena) { - return (iallocztm(tsd, size, false, try_tcache, false, arena)); + return (iallocztm(tsd, size, false, tcache, false, arena)); } JEMALLOC_ALWAYS_INLINE void * imalloc(tsd_t *tsd, size_t size) { - return (iallocztm(tsd, size, false, true, false, NULL)); + return (iallocztm(tsd, size, false, tcache_get(tsd, true), false, NULL)); } JEMALLOC_ALWAYS_INLINE void * -icalloct(tsd_t *tsd, size_t size, bool try_tcache, arena_t *arena) +icalloct(tsd_t *tsd, size_t size, tcache_t *tcache, arena_t *arena) { - return (iallocztm(tsd, size, true, try_tcache, false, arena)); + return (iallocztm(tsd, size, true, tcache, false, arena)); } JEMALLOC_ALWAYS_INLINE void * icalloc(tsd_t *tsd, size_t size) { - return (iallocztm(tsd, size, true, true, false, NULL)); + return (iallocztm(tsd, size, true, tcache_get(tsd, true), false, NULL)); } JEMALLOC_ALWAYS_INLINE void * ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, - bool try_tcache, bool is_metadata, arena_t *arena) + tcache_t *tcache, bool is_metadata, arena_t *arena) { void *ret; @@ -909,7 +925,7 @@ ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, assert(usize == sa2u(usize, alignment)); if (usize <= SMALL_MAXCLASS && alignment < PAGE) - ret = arena_malloc(tsd, arena, usize, zero, try_tcache); + ret = arena_malloc(tsd, arena, usize, zero, tcache); else { if (likely(usize <= arena_maxclass)) { arena = arena_choose(tsd, arena); @@ -917,10 +933,10 @@ ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, return (NULL); ret = arena_palloc(arena, usize, alignment, zero); } else if (likely(alignment <= chunksize)) - ret = huge_malloc(tsd, arena, usize, zero, try_tcache); + ret = huge_malloc(tsd, arena, usize, zero, tcache); else { ret = huge_palloc(tsd, arena, usize, alignment, zero, - try_tcache); + tcache); } } assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); @@ -932,19 +948,19 @@ ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, } JEMALLOC_ALWAYS_INLINE void * -ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena) +ipalloct(tsd_t *tsd, size_t usize, size_t alignment, bool zero, + tcache_t *tcache, arena_t *arena) { - return (ipallocztm(tsd, usize, alignment, zero, try_tcache, false, - arena)); + return (ipallocztm(tsd, usize, alignment, zero, tcache, false, arena)); } JEMALLOC_ALWAYS_INLINE void * ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) { - return (ipallocztm(tsd, usize, alignment, zero, true, false, NULL)); + return (ipallocztm(tsd, usize, alignment, zero, tcache_get(tsd, + NULL), false, NULL)); } JEMALLOC_ALWAYS_INLINE size_t @@ -981,7 +997,7 @@ p2rz(const void *ptr) } JEMALLOC_ALWAYS_INLINE void -idalloctm(tsd_t *tsd, void *ptr, bool try_tcache, bool is_metadata) +idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata) { arena_chunk_t *chunk; @@ -993,37 +1009,37 @@ idalloctm(tsd_t *tsd, void *ptr, bool try_tcache, bool is_metadata) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) - arena_dalloc(tsd, chunk, ptr, try_tcache); + arena_dalloc(tsd, chunk, ptr, tcache); else - huge_dalloc(tsd, ptr, try_tcache); + huge_dalloc(tsd, ptr, tcache); } JEMALLOC_ALWAYS_INLINE void -idalloct(tsd_t *tsd, void *ptr, bool try_tcache) +idalloct(tsd_t *tsd, void *ptr, tcache_t *tcache) { - idalloctm(tsd, ptr, try_tcache, false); + idalloctm(tsd, ptr, tcache, false); } JEMALLOC_ALWAYS_INLINE void idalloc(tsd_t *tsd, void *ptr) { - idalloctm(tsd, ptr, true, false); + idalloctm(tsd, ptr, tcache_get(tsd, false), false); } JEMALLOC_ALWAYS_INLINE void -iqalloc(tsd_t *tsd, void *ptr, bool try_tcache) +iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) { if (config_fill && unlikely(opt_quarantine)) quarantine(tsd, ptr); else - idalloctm(tsd, ptr, try_tcache, false); + idalloctm(tsd, ptr, tcache, false); } JEMALLOC_ALWAYS_INLINE void -isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) +isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) { arena_chunk_t *chunk; @@ -1031,25 +1047,24 @@ isdalloct(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) - arena_sdalloc(tsd, chunk, ptr, size, try_tcache); + arena_sdalloc(tsd, chunk, ptr, size, tcache); else - huge_dalloc(tsd, ptr, try_tcache); + huge_dalloc(tsd, ptr, tcache); } JEMALLOC_ALWAYS_INLINE void -isqalloc(tsd_t *tsd, void *ptr, size_t size, bool try_tcache) +isqalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) { if (config_fill && unlikely(opt_quarantine)) quarantine(tsd, ptr); else - isdalloct(tsd, ptr, size, try_tcache); + isdalloct(tsd, ptr, size, tcache); } JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, - bool try_tcache_dalloc, arena_t *arena) + size_t extra, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { void *p; size_t usize, copysize; @@ -1057,7 +1072,7 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, arena); + p = ipalloct(tsd, usize, alignment, zero, tcache, arena); if (p == NULL) { if (extra == 0) return (NULL); @@ -1065,8 +1080,7 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, usize = sa2u(size, alignment); if (usize == 0) return (NULL); - p = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, - arena); + p = ipalloct(tsd, usize, alignment, zero, tcache, arena); if (p == NULL) return (NULL); } @@ -1076,13 +1090,13 @@ iralloct_realign(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); - isqalloc(tsd, ptr, oldsize, try_tcache_dalloc); + isqalloc(tsd, ptr, oldsize, tcache); return (p); } JEMALLOC_ALWAYS_INLINE void * iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) + bool zero, tcache_t *tcache, arena_t *arena) { assert(ptr != NULL); @@ -1095,15 +1109,15 @@ iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, * and copy. */ return (iralloct_realign(tsd, ptr, oldsize, size, 0, alignment, - zero, try_tcache_alloc, try_tcache_dalloc, arena)); + zero, tcache, arena)); } if (likely(size <= arena_maxclass)) { return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, - alignment, zero, try_tcache_alloc, try_tcache_dalloc)); + alignment, zero, tcache)); } else { return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0, - alignment, zero, try_tcache_alloc, try_tcache_dalloc)); + alignment, zero, tcache)); } } @@ -1112,8 +1126,8 @@ iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, bool zero) { - return (iralloct(tsd, ptr, oldsize, size, alignment, zero, true, true, - NULL)); + return (iralloct(tsd, ptr, oldsize, size, alignment, zero, + tcache_get(tsd, true), NULL)); } JEMALLOC_ALWAYS_INLINE bool diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 7a78f580..cf42bead 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -425,6 +425,11 @@ tcache_get_hard tcache_maxclass tcache_salloc tcache_stats_merge +tcaches +tcaches_create +tcaches_destroy +tcaches_flush +tcaches_get thread_allocated_cleanup thread_deallocated_cleanup tsd_booted diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 6e97b3dd..2a3952be 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -4,6 +4,7 @@ typedef struct tcache_bin_info_s tcache_bin_info_t; typedef struct tcache_bin_s tcache_bin_t; typedef struct tcache_s tcache_t; +typedef struct tcaches_s tcaches_t; /* * tcache pointers close to NULL are used to encode state information that is @@ -70,7 +71,6 @@ struct tcache_bin_s { struct tcache_s { ql_elm(tcache_t) link; /* Used for aggregating stats. */ uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */ - arena_t *arena; /* This thread's arena. */ unsigned ev_cnt; /* Event count since incremental GC. */ index_t next_gc_bin; /* Next bin to GC. */ tcache_bin_t tbins[1]; /* Dynamically sized. */ @@ -82,6 +82,14 @@ struct tcache_s { */ }; +/* Linkage for list of available (previously used) explicit tcache IDs. */ +struct tcaches_s { + union { + tcache_t *tcache; + tcaches_t *next; + }; +}; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS @@ -95,27 +103,41 @@ extern tcache_bin_info_t *tcache_bin_info; * Number of tcache bins. There are NBINS small-object bins, plus 0 or more * large-object bins. */ -extern size_t nhbins; +extern size_t nhbins; /* Maximum cached size class. */ -extern size_t tcache_maxclass; +extern size_t tcache_maxclass; + +/* + * Explicit tcaches, managed via the tcache.{create,flush,destroy} mallctls and + * usable via the MALLOCX_TCACHE() flag. The automatic per thread tcaches are + * completely disjoint from this data structure. tcaches starts off as a sparse + * array, so it has no physical memory footprint until individual pages are + * touched. This allows the entire array to be allocated the first time an + * explicit tcache is created without a disproportionate impact on memory usage. + */ +extern tcaches_t *tcaches; size_t tcache_salloc(const void *ptr); -void tcache_event_hard(tcache_t *tcache); -void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, - index_t binind); -void tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, - tcache_t *tcache); -void tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, - tcache_t *tcache); +void tcache_event_hard(tsd_t *tsd, tcache_t *tcache); +void *tcache_alloc_small_hard(tsd_t *tsd, tcache_t *tcache, + tcache_bin_t *tbin, index_t binind); +void tcache_bin_flush_small(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, + unsigned rem, tcache_t *tcache); +void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, + unsigned rem, tcache_t *tcache); void tcache_arena_associate(tcache_t *tcache, arena_t *arena); -void tcache_arena_reassociate(tcache_t *tcache, arena_t *arena); -void tcache_arena_dissociate(tcache_t *tcache); +void tcache_arena_reassociate(tcache_t *tcache, arena_t *oldarena, + arena_t *newarena); +void tcache_arena_dissociate(tcache_t *tcache, arena_t *arena); tcache_t *tcache_get_hard(tsd_t *tsd); tcache_t *tcache_create(tsd_t *tsd, arena_t *arena); void tcache_cleanup(tsd_t *tsd); void tcache_enabled_cleanup(tsd_t *tsd); void tcache_stats_merge(tcache_t *tcache, arena_t *arena); +bool tcaches_create(tsd_t *tsd, unsigned *r_ind); +void tcaches_flush(tsd_t *tsd, unsigned ind); +void tcaches_destroy(tsd_t *tsd, unsigned ind); bool tcache_boot(void); #endif /* JEMALLOC_H_EXTERNS */ @@ -123,16 +145,21 @@ bool tcache_boot(void); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -void tcache_event(tcache_t *tcache); +void tcache_event(tsd_t *tsd, tcache_t *tcache); void tcache_flush(void); bool tcache_enabled_get(void); tcache_t *tcache_get(tsd_t *tsd, bool create); void tcache_enabled_set(bool enabled); void *tcache_alloc_easy(tcache_bin_t *tbin); -void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); -void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero); -void tcache_dalloc_small(tcache_t *tcache, void *ptr, index_t binind); -void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); +void *tcache_alloc_small(tsd_t *tsd, tcache_t *tcache, size_t size, + bool zero); +void *tcache_alloc_large(tsd_t *tsd, tcache_t *tcache, size_t size, + bool zero); +void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, + index_t binind); +void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, + size_t size); +tcache_t *tcaches_get(tsd_t *tsd, unsigned ind); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) @@ -202,7 +229,7 @@ tcache_get(tsd_t *tsd, bool create) } JEMALLOC_ALWAYS_INLINE void -tcache_event(tcache_t *tcache) +tcache_event(tsd_t *tsd, tcache_t *tcache) { if (TCACHE_GC_INCR == 0) @@ -211,7 +238,7 @@ tcache_event(tcache_t *tcache) tcache->ev_cnt++; assert(tcache->ev_cnt <= TCACHE_GC_INCR); if (unlikely(tcache->ev_cnt == TCACHE_GC_INCR)) - tcache_event_hard(tcache); + tcache_event_hard(tsd, tcache); } JEMALLOC_ALWAYS_INLINE void * @@ -231,7 +258,7 @@ tcache_alloc_easy(tcache_bin_t *tbin) } JEMALLOC_ALWAYS_INLINE void * -tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) +tcache_alloc_small(tsd_t *tsd, tcache_t *tcache, size_t size, bool zero) { void *ret; index_t binind; @@ -244,7 +271,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) usize = index2size(binind); ret = tcache_alloc_easy(tbin); if (unlikely(ret == NULL)) { - ret = tcache_alloc_small_hard(tcache, tbin, binind); + ret = tcache_alloc_small_hard(tsd, tcache, tbin, binind); if (ret == NULL) return (NULL); } @@ -270,12 +297,12 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) tbin->tstats.nrequests++; if (config_prof) tcache->prof_accumbytes += usize; - tcache_event(tcache); + tcache_event(tsd, tcache); return (ret); } JEMALLOC_ALWAYS_INLINE void * -tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) +tcache_alloc_large(tsd_t *tsd, tcache_t *tcache, size_t size, bool zero) { void *ret; index_t binind; @@ -293,7 +320,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) * Only allocate one large object at a time, because it's quite * expensive to create one and not use it. */ - ret = arena_malloc_large(tcache->arena, usize, zero); + ret = arena_malloc_large(arena_choose(tsd, NULL), usize, zero); if (ret == NULL) return (NULL); } else { @@ -321,12 +348,12 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) tcache->prof_accumbytes += usize; } - tcache_event(tcache); + tcache_event(tsd, tcache); return (ret); } JEMALLOC_ALWAYS_INLINE void -tcache_dalloc_small(tcache_t *tcache, void *ptr, index_t binind) +tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, index_t binind) { tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; @@ -339,18 +366,18 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr, index_t binind) tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; if (unlikely(tbin->ncached == tbin_info->ncached_max)) { - tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >> - 1), tcache); + tcache_bin_flush_small(tsd, tbin, binind, + (tbin_info->ncached_max >> 1), tcache); } assert(tbin->ncached < tbin_info->ncached_max); tbin->avail[tbin->ncached] = ptr; tbin->ncached++; - tcache_event(tcache); + tcache_event(tsd, tcache); } JEMALLOC_ALWAYS_INLINE void -tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) +tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, size_t size) { index_t binind; tcache_bin_t *tbin; @@ -368,14 +395,23 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; if (unlikely(tbin->ncached == tbin_info->ncached_max)) { - tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >> - 1), tcache); + tcache_bin_flush_large(tsd, tbin, binind, + (tbin_info->ncached_max >> 1), tcache); } assert(tbin->ncached < tbin_info->ncached_max); tbin->avail[tbin->ncached] = ptr; tbin->ncached++; - tcache_event(tcache); + tcache_event(tsd, tcache); +} + +JEMALLOC_ALWAYS_INLINE tcache_t * +tcaches_get(tsd_t *tsd, unsigned ind) +{ + tcaches_t *elm = &tcaches[ind]; + if (unlikely(elm->tcache == NULL)) + elm->tcache = tcache_create(tsd, arena_choose(tsd, NULL)); + return (elm->tcache); } #endif diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in index 99f12611..7d1dcf4a 100644 --- a/include/jemalloc/jemalloc_macros.h.in +++ b/include/jemalloc/jemalloc_macros.h.in @@ -19,8 +19,16 @@ ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) # endif # define MALLOCX_ZERO ((int)0x40) -/* Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". */ -# define MALLOCX_ARENA(a) ((int)(((a)+1) << 8)) +/* + * Bias tcache index bits so that 0 encodes "automatic tcache management", and 1 + * encodes MALLOCX_TCACHE_NONE. + */ +# define MALLOCX_TCACHE(tc) ((int)(((tc)+2) << 8)) +# define MALLOCX_TCACHE_NONE MALLOCX_TCACHE(-1) +/* + * Bias arena index bits so that 0 encodes "use an automatically chosen arena". + */ +# define MALLOCX_ARENA(a) ((int)(((a)+1) << 20)) #ifdef JEMALLOC_HAVE_ATTR # define JEMALLOC_ATTR(s) __attribute__((s)) diff --git a/src/arena.c b/src/arena.c index a5033bf8..907fbd7f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2182,8 +2182,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, void * arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, - bool try_tcache_dalloc) + size_t extra, size_t alignment, bool zero, tcache_t *tcache) { void *ret; size_t copysize; @@ -2201,12 +2200,9 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - ret = ipalloct(tsd, usize, alignment, zero, try_tcache_alloc, - arena); - } else { - ret = arena_malloc(tsd, arena, size + extra, zero, - try_tcache_alloc); - } + ret = ipalloct(tsd, usize, alignment, zero, tcache, arena); + } else + ret = arena_malloc(tsd, arena, size + extra, zero, tcache); if (ret == NULL) { if (extra == 0) @@ -2216,12 +2212,10 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t usize = sa2u(size, alignment); if (usize == 0) return (NULL); - ret = ipalloct(tsd, usize, alignment, zero, - try_tcache_alloc, arena); - } else { - ret = arena_malloc(tsd, arena, size, zero, - try_tcache_alloc); - } + ret = ipalloct(tsd, usize, alignment, zero, tcache, + arena); + } else + ret = arena_malloc(tsd, arena, size, zero, tcache); if (ret == NULL) return (NULL); @@ -2236,7 +2230,7 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, copysize = (size < oldsize) ? size : oldsize; JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); - isqalloc(tsd, ptr, oldsize, try_tcache_dalloc); + isqalloc(tsd, ptr, oldsize, tcache); return (ret); } diff --git a/src/ckh.c b/src/ckh.c index db2ae392..ad075d60 100644 --- a/src/ckh.c +++ b/src/ckh.c @@ -270,7 +270,8 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) ret = true; goto label_return; } - tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true); + tab = (ckhc_t *)ipalloct(tsd, usize, CACHELINE, true, NULL, + NULL); if (tab == NULL) { ret = true; goto label_return; @@ -313,7 +314,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); if (usize == 0) return; - tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true); + tab = (ckhc_t *)ipalloct(tsd, usize, CACHELINE, true, NULL, NULL); if (tab == NULL) { /* * An OOM error isn't worth propagating, since it doesn't @@ -389,7 +390,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ret = true; goto label_return; } - ckh->tab = (ckhc_t *)ipalloc(tsd, usize, CACHELINE, true); + ckh->tab = (ckhc_t *)ipalloct(tsd, usize, CACHELINE, true, NULL, NULL); if (ckh->tab == NULL) { ret = true; goto label_return; diff --git a/src/ctl.c b/src/ctl.c index 63a689a3..a2838032 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -110,6 +110,9 @@ CTL_PROTO(opt_prof_gdump) CTL_PROTO(opt_prof_final) CTL_PROTO(opt_prof_leak) CTL_PROTO(opt_prof_accum) +CTL_PROTO(tcache_create) +CTL_PROTO(tcache_flush) +CTL_PROTO(tcache_destroy) CTL_PROTO(arena_i_purge) static void arena_purge(unsigned arena_ind); CTL_PROTO(arena_i_dss) @@ -275,6 +278,12 @@ static const ctl_named_node_t opt_node[] = { {NAME("prof_accum"), CTL(opt_prof_accum)} }; +static const ctl_named_node_t tcache_node[] = { + {NAME("create"), CTL(tcache_create)}, + {NAME("flush"), CTL(tcache_flush)}, + {NAME("destroy"), CTL(tcache_destroy)} +}; + static const ctl_named_node_t chunk_node[] = { {NAME("alloc"), CTL(arena_i_chunk_alloc)}, {NAME("dalloc"), CTL(arena_i_chunk_dalloc)} @@ -474,6 +483,7 @@ static const ctl_named_node_t root_node[] = { {NAME("thread"), CHILD(named, thread)}, {NAME("config"), CHILD(named, config)}, {NAME("opt"), CHILD(named, opt)}, + {NAME("tcache"), CHILD(named, tcache)}, {NAME("arena"), CHILD(indexed, arena)}, {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)}, @@ -1281,19 +1291,21 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, { int ret; tsd_t *tsd; - arena_t *arena; + arena_t *oldarena; unsigned newind, oldind; tsd = tsd_fetch(); - arena = arena_choose(tsd, NULL); - if (arena == NULL) + oldarena = arena_choose(tsd, NULL); + if (oldarena == NULL) return (EAGAIN); malloc_mutex_lock(&ctl_mtx); - newind = oldind = arena->ind; + newind = oldind = oldarena->ind; WRITE(newind, unsigned); READ(oldind, unsigned); if (newind != oldind) { + arena_t *newarena; + if (newind >= ctl_stats.narenas) { /* New arena index is out of range. */ ret = EFAULT; @@ -1301,8 +1313,8 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, } /* Initialize arena if necessary. */ - arena = arena_get(tsd, newind, true, true); - if (arena == NULL) { + newarena = arena_get(tsd, newind, true, true); + if (newarena == NULL) { ret = EAGAIN; goto label_return; } @@ -1310,8 +1322,10 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, arena_migrate(tsd, oldind, newind); if (config_tcache) { tcache_t *tcache = tsd_tcache_get(tsd); - if (tcache != NULL) - tcache_arena_reassociate(tcache, arena); + if (tcache != NULL) { + tcache_arena_reassociate(tcache, oldarena, + newarena); + } } } @@ -1438,6 +1452,89 @@ label_return: /******************************************************************************/ +static int +tcache_create_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + tsd_t *tsd; + unsigned tcache_ind; + + if (!config_tcache) + return (ENOENT); + + tsd = tsd_fetch(); + + malloc_mutex_lock(&ctl_mtx); + READONLY(); + if (tcaches_create(tsd, &tcache_ind)) { + ret = EFAULT; + goto label_return; + } + READ(tcache_ind, unsigned); + + ret = 0; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +static int +tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + tsd_t *tsd; + unsigned tcache_ind; + + if (!config_tcache) + return (ENOENT); + + tsd = tsd_fetch(); + + WRITEONLY(); + tcache_ind = UINT_MAX; + WRITE(tcache_ind, unsigned); + if (tcache_ind == UINT_MAX) { + ret = EFAULT; + goto label_return; + } + tcaches_flush(tsd, tcache_ind); + + ret = 0; +label_return: + return (ret); +} + +static int +tcache_destroy_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + tsd_t *tsd; + unsigned tcache_ind; + + if (!config_tcache) + return (ENOENT); + + tsd = tsd_fetch(); + + WRITEONLY(); + tcache_ind = UINT_MAX; + WRITE(tcache_ind, unsigned); + if (tcache_ind == UINT_MAX) { + ret = EFAULT; + goto label_return; + } + tcaches_destroy(tsd, tcache_ind); + + ret = 0; +label_return: + return (ret); +} + +/******************************************************************************/ + /* ctl_mutex must be held during execution of this function. */ static void arena_purge(unsigned arena_ind) diff --git a/src/huge.c b/src/huge.c index 84a1ab23..db0ecd51 100644 --- a/src/huge.c +++ b/src/huge.c @@ -13,7 +13,8 @@ static malloc_mutex_t huge_mtx; static extent_tree_t huge; void * -huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, bool try_tcache) +huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, + tcache_t *tcache) { size_t usize; @@ -23,12 +24,12 @@ huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, bool try_tcache) return (NULL); } - return (huge_palloc(tsd, arena, usize, chunksize, zero, try_tcache)); + return (huge_palloc(tsd, arena, usize, chunksize, zero, tcache)); } void * huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, - bool zero, bool try_tcache) + bool zero, tcache_t *tcache) { void *ret; extent_node_t *node; @@ -38,7 +39,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, /* Allocate an extent node with which to track the chunk. */ node = ipallocztm(tsd, CACHELINE_CEILING(sizeof(extent_node_t)), - CACHELINE, false, try_tcache, true, arena); + CACHELINE, false, tcache, true, arena); if (node == NULL) return (NULL); @@ -50,7 +51,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, arena = arena_choose(tsd, arena); if (unlikely(arena == NULL) || (ret = arena_chunk_alloc_huge(arena, usize, alignment, &is_zeroed)) == NULL) { - idalloctm(tsd, node, try_tcache, true); + idalloctm(tsd, node, tcache, true); return (NULL); } @@ -307,8 +308,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, void * huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, - bool try_tcache_dalloc) + size_t extra, size_t alignment, bool zero, tcache_t *tcache) { void *ret; size_t copysize; @@ -324,11 +324,9 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, */ if (alignment > chunksize) { ret = huge_palloc(tsd, arena, size + extra, alignment, zero, - try_tcache_alloc); - } else { - ret = huge_malloc(tsd, arena, size + extra, zero, - try_tcache_alloc); - } + tcache); + } else + ret = huge_malloc(tsd, arena, size + extra, zero, tcache); if (ret == NULL) { if (extra == 0) @@ -336,11 +334,9 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, /* Try again, this time without extra. */ if (alignment > chunksize) { ret = huge_palloc(tsd, arena, size, alignment, zero, - try_tcache_alloc); - } else { - ret = huge_malloc(tsd, arena, size, zero, - try_tcache_alloc); - } + tcache); + } else + ret = huge_malloc(tsd, arena, size, zero, tcache); if (ret == NULL) return (NULL); @@ -352,12 +348,12 @@ huge_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); - isqalloc(tsd, ptr, oldsize, try_tcache_dalloc); + isqalloc(tsd, ptr, oldsize, tcache); return (ret); } void -huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache) +huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) { extent_node_t *node; @@ -368,7 +364,7 @@ huge_dalloc(tsd_t *tsd, void *ptr, bool try_tcache) huge_dalloc_junk(node->addr, node->size); arena_chunk_dalloc_huge(node->arena, node->addr, node->size); - idalloctm(tsd, node, try_tcache, true); + idalloctm(tsd, node, tcache, true); } arena_t * diff --git a/src/jemalloc.c b/src/jemalloc.c index d1fa674c..94477914 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -367,6 +367,8 @@ arena_init_locked(unsigned ind) /* Expand arenas if necessary. */ assert(ind <= narenas_total); + if (ind > MALLOCX_ARENA_MAX) + return (NULL); if (ind == narenas_total) { unsigned narenas_new = narenas_total + 1; arena_t **arenas_new = @@ -1696,7 +1698,7 @@ irealloc_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t usize) } JEMALLOC_INLINE_C void -ifree(tsd_t *tsd, void *ptr, bool try_tcache) +ifree(tsd_t *tsd, void *ptr, tcache_t *tcache) { size_t usize; UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); @@ -1713,12 +1715,12 @@ ifree(tsd_t *tsd, void *ptr, bool try_tcache) *tsd_thread_deallocatedp_get(tsd) += usize; if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); - iqalloc(tsd, ptr, try_tcache); + iqalloc(tsd, ptr, tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); } JEMALLOC_INLINE_C void -isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache) +isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache) { UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); @@ -1731,7 +1733,7 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, bool try_tcache) *tsd_thread_deallocatedp_get(tsd) += usize; if (config_valgrind && unlikely(in_valgrind)) rzsize = p2rz(ptr); - isqalloc(tsd, ptr, usize, try_tcache); + isqalloc(tsd, ptr, usize, tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); } @@ -1749,7 +1751,7 @@ je_realloc(void *ptr, size_t size) /* realloc(ptr, 0) is equivalent to free(ptr). */ UTRACE(ptr, 0, 0); tsd = tsd_fetch(); - ifree(tsd, ptr, true); + ifree(tsd, ptr, tcache_get(tsd, false)); return (NULL); } size = 1; @@ -1802,8 +1804,10 @@ je_free(void *ptr) { UTRACE(ptr, 0, 0); - if (likely(ptr != NULL)) - ifree(tsd_fetch(), ptr, true); + if (likely(ptr != NULL)) { + tsd_t *tsd = tsd_fetch(); + ifree(tsd, ptr, tcache_get(tsd, false)); + } } /* @@ -1875,7 +1879,7 @@ JEMALLOC_EXPORT void *(*__memalign_hook)(size_t alignment, size_t size) = JEMALLOC_ALWAYS_INLINE_C bool imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize, - size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) + size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) { if ((flags & MALLOCX_LG_ALIGN_MASK) == 0) { @@ -1886,22 +1890,26 @@ imallocx_flags_decode_hard(tsd_t *tsd, size_t size, int flags, size_t *usize, *usize = sa2u(size, *alignment); } *zero = MALLOCX_ZERO_GET(flags); + if ((flags & MALLOCX_TCACHE_MASK) != 0) { + if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) + *tcache = NULL; + else + *tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); + } else + *tcache = tcache_get(tsd, true); if ((flags & MALLOCX_ARENA_MASK) != 0) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); - *try_tcache = false; *arena = arena_get(tsd, arena_ind, true, true); if (unlikely(*arena == NULL)) return (true); - } else { - *try_tcache = true; + } else *arena = NULL; - } return (false); } JEMALLOC_ALWAYS_INLINE_C bool imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, - size_t *alignment, bool *zero, bool *try_tcache, arena_t **arena) + size_t *alignment, bool *zero, tcache_t **tcache, arena_t **arena) { if (likely(flags == 0)) { @@ -1909,55 +1917,53 @@ imallocx_flags_decode(tsd_t *tsd, size_t size, int flags, size_t *usize, assert(usize != 0); *alignment = 0; *zero = false; - *try_tcache = true; + *tcache = tcache_get(tsd, true); *arena = NULL; return (false); } else { return (imallocx_flags_decode_hard(tsd, size, flags, usize, - alignment, zero, try_tcache, arena)); + alignment, zero, tcache, arena)); } } JEMALLOC_ALWAYS_INLINE_C void * imallocx_flags(tsd_t *tsd, size_t usize, size_t alignment, bool zero, - bool try_tcache, arena_t *arena) + tcache_t *tcache, arena_t *arena) { - if (alignment != 0) { - return (ipalloct(tsd, usize, alignment, zero, try_tcache, - arena)); - } + if (alignment != 0) + return (ipalloct(tsd, usize, alignment, zero, tcache, arena)); if (zero) - return (icalloct(tsd, usize, try_tcache, arena)); - return (imalloct(tsd, usize, try_tcache, arena)); + return (icalloct(tsd, usize, tcache, arena)); + return (imalloct(tsd, usize, tcache, arena)); } JEMALLOC_ALWAYS_INLINE_C void * imallocx_maybe_flags(tsd_t *tsd, size_t size, int flags, size_t usize, - size_t alignment, bool zero, bool try_tcache, arena_t *arena) + size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { if (likely(flags == 0)) return (imalloc(tsd, size)); - return (imallocx_flags(tsd, usize, alignment, zero, try_tcache, arena)); + return (imallocx_flags(tsd, usize, alignment, zero, tcache, arena)); } static void * imallocx_prof_sample(tsd_t *tsd, size_t size, int flags, size_t usize, - size_t alignment, bool zero, bool try_tcache, arena_t *arena) + size_t alignment, bool zero, tcache_t *tcache, arena_t *arena) { void *p; if (usize <= SMALL_MAXCLASS) { assert(((alignment == 0) ? s2u(LARGE_MINCLASS) : sa2u(LARGE_MINCLASS, alignment)) == LARGE_MINCLASS); - p = imalloct(tsd, LARGE_MINCLASS, try_tcache, arena); + p = imalloct(tsd, LARGE_MINCLASS, tcache, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else { p = imallocx_maybe_flags(tsd, size, flags, usize, alignment, - zero, try_tcache, arena); + zero, tcache, arena); } return (p); @@ -1969,20 +1975,20 @@ imallocx_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) void *p; size_t alignment; bool zero; - bool try_tcache; + tcache_t *tcache; arena_t *arena; prof_tctx_t *tctx; if (unlikely(imallocx_flags_decode(tsd, size, flags, usize, &alignment, - &zero, &try_tcache, &arena))) + &zero, &tcache, &arena))) return (NULL); tctx = prof_alloc_prep(tsd, *usize, true); if (likely((uintptr_t)tctx == (uintptr_t)1U)) { p = imallocx_maybe_flags(tsd, size, flags, *usize, alignment, - zero, try_tcache, arena); + zero, tcache, arena); } else if ((uintptr_t)tctx > (uintptr_t)1U) { p = imallocx_prof_sample(tsd, size, flags, *usize, alignment, - zero, try_tcache, arena); + zero, tcache, arena); } else p = NULL; if (unlikely(p == NULL)) { @@ -1999,7 +2005,7 @@ imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) { size_t alignment; bool zero; - bool try_tcache; + tcache_t *tcache; arena_t *arena; if (likely(flags == 0)) { @@ -2009,10 +2015,9 @@ imallocx_no_prof(tsd_t *tsd, size_t size, int flags, size_t *usize) } if (unlikely(imallocx_flags_decode_hard(tsd, size, flags, usize, - &alignment, &zero, &try_tcache, &arena))) + &alignment, &zero, &tcache, &arena))) return (NULL); - return (imallocx_flags(tsd, *usize, alignment, zero, try_tcache, - arena)); + return (imallocx_flags(tsd, *usize, alignment, zero, tcache, arena)); } void * @@ -2053,8 +2058,8 @@ label_oom: static void * irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, - size_t alignment, size_t usize, bool zero, bool try_tcache_alloc, - bool try_tcache_dalloc, arena_t *arena, prof_tctx_t *tctx) + size_t alignment, size_t usize, bool zero, tcache_t *tcache, arena_t *arena, + prof_tctx_t *tctx) { void *p; @@ -2062,13 +2067,13 @@ irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, return (NULL); if (usize <= SMALL_MAXCLASS) { p = iralloct(tsd, oldptr, old_usize, LARGE_MINCLASS, alignment, - zero, try_tcache_alloc, try_tcache_dalloc, arena); + zero, tcache, arena); if (p == NULL) return (NULL); arena_prof_promoted(p, usize); } else { p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, - try_tcache_alloc, try_tcache_dalloc, arena); + tcache, arena); } return (p); @@ -2076,8 +2081,8 @@ irallocx_prof_sample(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, JEMALLOC_ALWAYS_INLINE_C void * irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, - size_t alignment, size_t *usize, bool zero, bool try_tcache_alloc, - bool try_tcache_dalloc, arena_t *arena) + size_t alignment, size_t *usize, bool zero, tcache_t *tcache, + arena_t *arena) { void *p; prof_tctx_t *old_tctx, *tctx; @@ -2086,11 +2091,10 @@ irallocx_prof(tsd_t *tsd, void *oldptr, size_t old_usize, size_t size, tctx = prof_alloc_prep(tsd, *usize, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(tsd, oldptr, old_usize, size, - alignment, *usize, zero, try_tcache_alloc, - try_tcache_dalloc, arena, tctx); + alignment, *usize, zero, tcache, arena, tctx); } else { p = iralloct(tsd, oldptr, old_usize, size, alignment, zero, - try_tcache_alloc, try_tcache_dalloc, arena); + tcache, arena); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, false); @@ -2123,8 +2127,8 @@ je_rallocx(void *ptr, size_t size, int flags) UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t alignment = MALLOCX_ALIGN_GET(flags); bool zero = flags & MALLOCX_ZERO; - bool try_tcache_alloc, try_tcache_dalloc; arena_t *arena; + tcache_t *tcache; assert(ptr != NULL); assert(size != 0); @@ -2134,18 +2138,19 @@ je_rallocx(void *ptr, size_t size, int flags) if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { unsigned arena_ind = MALLOCX_ARENA_GET(flags); - arena_chunk_t *chunk; - try_tcache_alloc = false; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena = arena_get(tsd, arena_ind, true, true); if (unlikely(arena == NULL)) goto label_oom; - try_tcache_dalloc = (chunk == ptr || chunk->arena != arena); - } else { - try_tcache_alloc = true; - try_tcache_dalloc = true; + } else arena = NULL; - } + + if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { + if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) + tcache = NULL; + else + tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); + } else + tcache = tcache_get(tsd, true); old_usize = isalloc(ptr, config_prof); if (config_valgrind && unlikely(in_valgrind)) @@ -2155,12 +2160,12 @@ je_rallocx(void *ptr, size_t size, int flags) usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); assert(usize != 0); p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, - zero, try_tcache_alloc, try_tcache_dalloc, arena); + zero, tcache, arena); if (unlikely(p == NULL)) goto label_oom; } else { p = iralloct(tsd, ptr, old_usize, size, alignment, zero, - try_tcache_alloc, try_tcache_dalloc, arena); + tcache, arena); if (unlikely(p == NULL)) goto label_oom; if (config_stats || (config_valgrind && unlikely(in_valgrind))) @@ -2319,28 +2324,22 @@ void je_dallocx(void *ptr, int flags) { tsd_t *tsd; - bool try_tcache; + tcache_t *tcache; assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); tsd = tsd_fetch(); - if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { - unsigned arena_ind = MALLOCX_ARENA_GET(flags); - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena_t *arena = arena_get(tsd, arena_ind, true, true); - /* - * If arena is NULL, the application passed an arena that has - * never been used before, which is unsupported during - * deallocation. - */ - assert(arena != NULL); - try_tcache = (chunk == ptr || chunk->arena != arena); + if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { + if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) + tcache = NULL; + else + tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); } else - try_tcache = true; + tcache = tcache_get(tsd, false); UTRACE(ptr, 0, 0); - ifree(tsd_fetch(), ptr, try_tcache); + ifree(tsd_fetch(), ptr, tcache); } JEMALLOC_ALWAYS_INLINE_C size_t @@ -2360,7 +2359,7 @@ void je_sdallocx(void *ptr, size_t size, int flags) { tsd_t *tsd; - bool try_tcache; + tcache_t *tcache; size_t usize; assert(ptr != NULL); @@ -2369,21 +2368,16 @@ je_sdallocx(void *ptr, size_t size, int flags) assert(usize == isalloc(ptr, config_prof)); tsd = tsd_fetch(); - if (unlikely((flags & MALLOCX_ARENA_MASK) != 0)) { - unsigned arena_ind = MALLOCX_ARENA_GET(flags); - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena_t *arena = arena_get(tsd, arena_ind, true, true); - /* - * If arena is NULL, the application passed an arena that has - * never been used before, which is unsupported during - * deallocation. - */ - try_tcache = (chunk == ptr || chunk->arena != arena); + if (unlikely((flags & MALLOCX_TCACHE_MASK) != 0)) { + if ((flags & MALLOCX_TCACHE_MASK) == MALLOCX_TCACHE_NONE) + tcache = NULL; + else + tcache = tcaches_get(tsd, MALLOCX_TCACHE_GET(flags)); } else - try_tcache = true; + tcache = tcache_get(tsd, false); UTRACE(ptr, 0, 0); - isfree(tsd, ptr, usize, try_tcache); + isfree(tsd, ptr, usize, tcache); } size_t diff --git a/src/prof.c b/src/prof.c index 04b2591c..4f1580b0 100644 --- a/src/prof.c +++ b/src/prof.c @@ -540,7 +540,8 @@ prof_gctx_create(tsd_t *tsd, prof_bt_t *bt) * Create a single allocation that has space for vec of length bt->len. */ prof_gctx_t *gctx = (prof_gctx_t *)iallocztm(tsd, offsetof(prof_gctx_t, - vec) + (bt->len * sizeof(void *)), false, true, true, NULL); + vec) + (bt->len * sizeof(void *)), false, tcache_get(tsd, true), + true, NULL); if (gctx == NULL) return (NULL); gctx->lock = prof_gctx_mutex_choose(); @@ -581,7 +582,7 @@ prof_gctx_try_destroy(tsd_t *tsd, prof_tdata_t *tdata_self, prof_gctx_t *gctx, prof_leave(tsd, tdata_self); /* Destroy gctx. */ malloc_mutex_unlock(gctx->lock); - idalloctm(tsd, gctx, true, true); + idalloctm(tsd, gctx, tcache_get(tsd, false), true); } else { /* * Compensate for increment in prof_tctx_destroy() or @@ -681,7 +682,7 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) prof_tdata_destroy(tsd, tdata, false); if (destroy_tctx) - idalloctm(tsd, tctx, true, true); + idalloctm(tsd, tctx, tcache_get(tsd, false), true); } static bool @@ -710,7 +711,7 @@ prof_lookup_global(tsd_t *tsd, prof_bt_t *bt, prof_tdata_t *tdata, if (ckh_insert(tsd, &bt2gctx, btkey.v, gctx.v)) { /* OOM. */ prof_leave(tsd, tdata); - idalloctm(tsd, gctx.v, true, true); + idalloctm(tsd, gctx.v, tcache_get(tsd, false), true); return (true); } new_gctx = true; @@ -754,6 +755,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) ret.p->prepared = true; malloc_mutex_unlock(tdata->lock); if (not_found) { + tcache_t *tcache; void *btkey; prof_gctx_t *gctx; bool new_gctx, error; @@ -767,7 +769,8 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) return (NULL); /* Link a prof_tctx_t into gctx for this thread. */ - ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, true, true, + tcache = tcache_get(tsd, true); + ret.v = iallocztm(tsd, sizeof(prof_tctx_t), false, tcache, true, NULL); if (ret.p == NULL) { if (new_gctx) @@ -786,7 +789,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) if (error) { if (new_gctx) prof_gctx_try_destroy(tsd, tdata, gctx, tdata); - idalloctm(tsd, ret.v, true, true); + idalloctm(tsd, ret.v, tcache, true); return (NULL); } malloc_mutex_lock(gctx->lock); @@ -1166,7 +1169,8 @@ prof_gctx_finish(tsd_t *tsd, prof_gctx_tree_t *gctxs) to_destroy); tctx_tree_remove(&gctx->tctxs, to_destroy); - idalloctm(tsd, to_destroy, true, true); + idalloctm(tsd, to_destroy, + tcache_get(tsd, false), true); } else next = NULL; } while (next != NULL); @@ -1644,12 +1648,14 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, char *thread_name, bool active) { prof_tdata_t *tdata; + tcache_t *tcache; cassert(config_prof); /* Initialize an empty cache for this thread. */ + tcache = tcache_get(tsd, true); tdata = (prof_tdata_t *)iallocztm(tsd, sizeof(prof_tdata_t), false, - true, true, NULL); + tcache, true, NULL); if (tdata == NULL) return (NULL); @@ -1662,7 +1668,7 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) { - idalloctm(tsd, tdata, true, true); + idalloctm(tsd, tdata, tcache, true); return (NULL); } @@ -1708,16 +1714,18 @@ static void prof_tdata_destroy_locked(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached) { + tcache_t *tcache; assert(prof_tdata_should_destroy(tdata, even_if_attached)); assert(tsd_prof_tdata_get(tsd) != tdata); tdata_tree_remove(&tdatas, tdata); + tcache = tcache_get(tsd, false); if (tdata->thread_name != NULL) - idalloctm(tsd, tdata->thread_name, true, true); + idalloctm(tsd, tdata->thread_name, tcache, true); ckh_delete(tsd, &tdata->bt2tctx); - idalloctm(tsd, tdata, true, true); + idalloctm(tsd, tdata, tcache, true); } static void @@ -1878,7 +1886,7 @@ prof_thread_name_alloc(tsd_t *tsd, const char *thread_name) if (size == 1) return (""); - ret = iallocztm(tsd, size, false, true, true, NULL); + ret = iallocztm(tsd, size, false, tcache_get(tsd, true), true, NULL); if (ret == NULL) return (NULL); memcpy(ret, thread_name, size); @@ -1910,7 +1918,8 @@ prof_thread_name_set(tsd_t *tsd, const char *thread_name) return (EAGAIN); if (tdata->thread_name != NULL) { - idalloctm(tsd, tdata->thread_name, true, true); + idalloctm(tsd, tdata->thread_name, tcache_get(tsd, false), + true); tdata->thread_name = NULL; } if (strlen(s) > 0) diff --git a/src/quarantine.c b/src/quarantine.c index 094b44d3..adc7305d 100644 --- a/src/quarantine.c +++ b/src/quarantine.c @@ -27,8 +27,8 @@ quarantine_init(tsd_t *tsd, size_t lg_maxobjs) assert(tsd_nominal(tsd)); quarantine = (quarantine_t *)iallocztm(tsd, offsetof(quarantine_t, objs) - + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)), false, true, - true, NULL); + + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t)), false, + tcache_get(tsd, true), true, NULL); if (quarantine == NULL) return (NULL); quarantine->curbytes = 0; @@ -55,7 +55,7 @@ quarantine_alloc_hook_work(tsd_t *tsd) if (tsd_quarantine_get(tsd) == NULL) tsd_quarantine_set(tsd, quarantine); else - idalloctm(tsd, quarantine, true, true); + idalloctm(tsd, quarantine, tcache_get(tsd, false), true); } static quarantine_t * @@ -87,7 +87,7 @@ quarantine_grow(tsd_t *tsd, quarantine_t *quarantine) memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * sizeof(quarantine_obj_t)); } - idalloctm(tsd, quarantine, true, true); + idalloctm(tsd, quarantine, tcache_get(tsd, false), true); tsd_quarantine_set(tsd, ret); return (ret); @@ -177,7 +177,7 @@ quarantine_cleanup(tsd_t *tsd) quarantine = tsd_quarantine_get(tsd); if (quarantine != NULL) { quarantine_drain(tsd, quarantine, 0); - idalloctm(tsd, quarantine, true, true); + idalloctm(tsd, quarantine, tcache_get(tsd, false), true); tsd_quarantine_set(tsd, NULL); } } diff --git a/src/tcache.c b/src/tcache.c index d638015f..c7d4f784 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -13,6 +13,14 @@ static unsigned stack_nelms; /* Total stack elms per tcache. */ size_t nhbins; size_t tcache_maxclass; +tcaches_t *tcaches; + +/* Index of first element within tcaches that has never been used. */ +static unsigned tcaches_past; + +/* Head of singly linked list tracking available tcaches elements. */ +static tcaches_t *tcaches_avail; + /******************************************************************************/ size_t tcache_salloc(const void *ptr) @@ -22,7 +30,7 @@ size_t tcache_salloc(const void *ptr) } void -tcache_event_hard(tcache_t *tcache) +tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { index_t binind = tcache->next_gc_bin; tcache_bin_t *tbin = &tcache->tbins[binind]; @@ -33,11 +41,11 @@ tcache_event_hard(tcache_t *tcache) * Flush (ceiling) 3/4 of the objects below the low water mark. */ if (binind < NBINS) { - tcache_bin_flush_small(tbin, binind, tbin->ncached - - tbin->low_water + (tbin->low_water >> 2), tcache); + tcache_bin_flush_small(tsd, tbin, binind, tbin->ncached + - tbin->low_water + (tbin->low_water >> 2), tcache); } else { - tcache_bin_flush_large(tbin, binind, tbin->ncached - - tbin->low_water + (tbin->low_water >> 2), tcache); + tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached + - tbin->low_water + (tbin->low_water >> 2), tcache); } /* * Reduce fill count by 2X. Limit lg_fill_div such that the @@ -62,11 +70,12 @@ tcache_event_hard(tcache_t *tcache) } void * -tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, index_t binind) +tcache_alloc_small_hard(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, + index_t binind) { void *ret; - arena_tcache_fill_small(tcache->arena, tbin, binind, + arena_tcache_fill_small(arena_choose(tsd, NULL), tbin, binind, config_prof ? tcache->prof_accumbytes : 0); if (config_prof) tcache->prof_accumbytes = 0; @@ -76,9 +85,10 @@ tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, index_t binind) } void -tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, - tcache_t *tcache) +tcache_bin_flush_small(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, + unsigned rem, tcache_t *tcache) { + arena_t *arena; void *ptr; unsigned i, nflush, ndeferred; bool merged_stats = false; @@ -86,21 +96,23 @@ tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, assert(binind < NBINS); assert(rem <= tbin->ncached); + arena = arena_choose(tsd, NULL); + assert(arena != NULL); for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { /* Lock the arena bin associated with the first object. */ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); - arena_t *arena = chunk->arena; + arena_t *bin_arena = chunk->arena; arena_bin_t *bin = &arena->bins[binind]; - if (config_prof && arena == tcache->arena) { + if (config_prof && bin_arena == arena) { if (arena_prof_accum(arena, tcache->prof_accumbytes)) prof_idump(); tcache->prof_accumbytes = 0; } malloc_mutex_lock(&bin->lock); - if (config_stats && arena == tcache->arena) { + if (config_stats && bin_arena == arena) { assert(!merged_stats); merged_stats = true; bin->stats.nflushes++; @@ -112,12 +124,12 @@ tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->arena == arena) { + if (chunk->arena == bin_arena) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_bits_t *bitselm = arena_bitselm_get(chunk, pageind); - arena_dalloc_bin_junked_locked(arena, chunk, + arena_dalloc_bin_junked_locked(bin_arena, chunk, ptr, bitselm); } else { /* @@ -137,7 +149,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - arena_bin_t *bin = &tcache->arena->bins[binind]; + arena_bin_t *bin = &arena->bins[binind]; malloc_mutex_lock(&bin->lock); bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; @@ -153,9 +165,10 @@ tcache_bin_flush_small(tcache_bin_t *tbin, index_t binind, unsigned rem, } void -tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, - tcache_t *tcache) +tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, + unsigned rem, tcache_t *tcache) { + arena_t *arena; void *ptr; unsigned i, nflush, ndeferred; bool merged_stats = false; @@ -163,17 +176,19 @@ tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, assert(binind < nhbins); assert(rem <= tbin->ncached); + arena = arena_choose(tsd, NULL); + assert(arena != NULL); for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { /* Lock the arena associated with the first object. */ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); - arena_t *arena = chunk->arena; + arena_t *locked_arena = chunk->arena; UNUSED bool idump; if (config_prof) idump = false; - malloc_mutex_lock(&arena->lock); - if ((config_prof || config_stats) && arena == tcache->arena) { + malloc_mutex_lock(&locked_arena->lock); + if ((config_prof || config_stats) && locked_arena == arena) { if (config_prof) { idump = arena_prof_accum_locked(arena, tcache->prof_accumbytes); @@ -193,9 +208,9 @@ tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->arena == arena) { - arena_dalloc_large_junked_locked(arena, chunk, - ptr); + if (chunk->arena == locked_arena) { + arena_dalloc_large_junked_locked(locked_arena, + chunk, ptr); } else { /* * This object was allocated via a different @@ -207,7 +222,7 @@ tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, ndeferred++; } } - malloc_mutex_unlock(&arena->lock); + malloc_mutex_unlock(&locked_arena->lock); if (config_prof && idump) prof_idump(); } @@ -216,7 +231,6 @@ tcache_bin_flush_large(tcache_bin_t *tbin, index_t binind, unsigned rem, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - arena_t *arena = tcache->arena; malloc_mutex_lock(&arena->lock); arena->stats.nrequests_large += tbin->tstats.nrequests; arena->stats.lstats[binind - NBINS].nrequests += @@ -243,27 +257,37 @@ tcache_arena_associate(tcache_t *tcache, arena_t *arena) ql_tail_insert(&arena->tcache_ql, tcache, link); malloc_mutex_unlock(&arena->lock); } - tcache->arena = arena; } void -tcache_arena_reassociate(tcache_t *tcache, arena_t *arena) +tcache_arena_reassociate(tcache_t *tcache, arena_t *oldarena, arena_t *newarena) { - tcache_arena_dissociate(tcache); - tcache_arena_associate(tcache, arena); + tcache_arena_dissociate(tcache, oldarena); + tcache_arena_associate(tcache, newarena); } void -tcache_arena_dissociate(tcache_t *tcache) +tcache_arena_dissociate(tcache_t *tcache, arena_t *arena) { if (config_stats) { /* Unlink from list of extant tcaches. */ - malloc_mutex_lock(&tcache->arena->lock); - ql_remove(&tcache->arena->tcache_ql, tcache, link); - tcache_stats_merge(tcache, tcache->arena); - malloc_mutex_unlock(&tcache->arena->lock); + malloc_mutex_lock(&arena->lock); + if (config_debug) { + bool in_ql = false; + tcache_t *iter; + ql_foreach(iter, &arena->tcache_ql, link) { + if (iter == tcache) { + in_ql = true; + break; + } + } + assert(in_ql); + } + ql_remove(&arena->tcache_ql, tcache, link); + tcache_stats_merge(tcache, arena); + malloc_mutex_unlock(&arena->lock); } } @@ -298,7 +322,7 @@ tcache_create(tsd_t *tsd, arena_t *arena) /* Avoid false cacheline sharing. */ size = sa2u(size, CACHELINE); - tcache = ipallocztm(tsd, size, CACHELINE, true, false, true, arena); + tcache = ipallocztm(tsd, size, CACHELINE, true, false, true, a0get()); if (tcache == NULL) return (NULL); @@ -318,16 +342,17 @@ tcache_create(tsd_t *tsd, arena_t *arena) static void tcache_destroy(tsd_t *tsd, tcache_t *tcache) { + arena_t *arena; unsigned i; - tcache_arena_dissociate(tcache); + arena = arena_choose(tsd, NULL); + tcache_arena_dissociate(tcache, arena); for (i = 0; i < NBINS; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_small(tbin, i, 0, tcache); + tcache_bin_flush_small(tsd, tbin, i, 0, tcache); if (config_stats && tbin->tstats.nrequests != 0) { - arena_t *arena = tcache->arena; arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(&bin->lock); bin->stats.nrequests += tbin->tstats.nrequests; @@ -337,10 +362,9 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) for (; i < nhbins; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_large(tbin, i, 0, tcache); + tcache_bin_flush_large(tsd, tbin, i, 0, tcache); if (config_stats && tbin->tstats.nrequests != 0) { - arena_t *arena = tcache->arena; malloc_mutex_lock(&arena->lock); arena->stats.nrequests_large += tbin->tstats.nrequests; arena->stats.lstats[i - NBINS].nrequests += @@ -350,7 +374,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) } if (config_prof && tcache->prof_accumbytes > 0 && - arena_prof_accum(tcache->arena, tcache->prof_accumbytes)) + arena_prof_accum(arena, tcache->prof_accumbytes)) prof_idump(); idalloctm(tsd, tcache, false, true); @@ -404,6 +428,66 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena) } } +bool +tcaches_create(tsd_t *tsd, unsigned *r_ind) +{ + tcache_t *tcache; + tcaches_t *elm; + + if (tcaches == NULL) { + tcaches = base_alloc(sizeof(tcache_t *) * + (MALLOCX_TCACHE_MAX+1)); + if (tcaches == NULL) + return (true); + } + + if (tcaches_avail == NULL && tcaches_past > MALLOCX_TCACHE_MAX) + return (true); + tcache = tcache_create(tsd, a0get()); + if (tcache == NULL) + return (true); + + if (tcaches_avail != NULL) { + elm = tcaches_avail; + tcaches_avail = tcaches_avail->next; + elm->tcache = tcache; + *r_ind = (elm - tcaches) / sizeof(tcaches_t); + } else { + elm = &tcaches[tcaches_past]; + elm->tcache = tcache; + *r_ind = tcaches_past; + tcaches_past++; + } + + return (false); +} + +static void +tcaches_elm_flush(tsd_t *tsd, tcaches_t *elm) +{ + + if (elm->tcache == NULL) + return; + tcache_destroy(tsd, elm->tcache); + elm->tcache = NULL; +} + +void +tcaches_flush(tsd_t *tsd, unsigned ind) +{ + + tcaches_elm_flush(tsd, &tcaches[ind]); +} + +void +tcaches_destroy(tsd_t *tsd, unsigned ind) +{ + tcaches_t *elm = &tcaches[ind]; + tcaches_elm_flush(tsd, elm); + elm->next = tcaches_avail; + tcaches_avail = elm; +} + bool tcache_boot(void) { diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index f4b7d1ab..10a6fcd6 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -211,6 +211,114 @@ TEST_BEGIN(test_manpage_example) } TEST_END +TEST_BEGIN(test_tcache_none) +{ + void *p0, *q, *p1; + + test_skip_if(!config_tcache); + + /* Allocate p and q. */ + p0 = mallocx(42, 0); + assert_ptr_not_null(p0, "Unexpected mallocx() failure"); + q = mallocx(42, 0); + assert_ptr_not_null(q, "Unexpected mallocx() failure"); + + /* Deallocate p and q, but bypass the tcache for q. */ + dallocx(p0, 0); + dallocx(q, MALLOCX_TCACHE_NONE); + + /* Make sure that tcache-based allocation returns p, not q. */ + p1 = mallocx(42, 0); + assert_ptr_not_null(p1, "Unexpected mallocx() failure"); + assert_ptr_eq(p0, p1, "Expected tcache to allocate cached region"); + + /* Clean up. */ + dallocx(p1, MALLOCX_TCACHE_NONE); +} +TEST_END + +TEST_BEGIN(test_tcache) +{ +#define NTCACHES 10 + unsigned tis[NTCACHES]; + void *ps[NTCACHES]; + void *qs[NTCACHES]; + unsigned i; + size_t sz, psz, qsz; + + test_skip_if(!config_tcache); + + psz = 42; + qsz = nallocx(psz, 0) + 1; + + /* Create tcaches. */ + for (i = 0; i < NTCACHES; i++) { + sz = sizeof(unsigned); + assert_d_eq(mallctl("tcache.create", &tis[i], &sz, NULL, 0), 0, + "Unexpected mallctl() failure, i=%u", i); + } + + /* Flush empty tcaches. */ + for (i = 0; i < NTCACHES; i++) { + assert_d_eq(mallctl("tcache.flush", NULL, NULL, &tis[i], + sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u", + i); + } + + /* Cache some allocations. */ + for (i = 0; i < NTCACHES; i++) { + ps[i] = mallocx(psz, MALLOCX_TCACHE(tis[i])); + assert_ptr_not_null(ps[i], "Unexpected mallocx() failure, i=%u", + i); + dallocx(ps[i], MALLOCX_TCACHE(tis[i])); + + qs[i] = mallocx(qsz, MALLOCX_TCACHE(tis[i])); + assert_ptr_not_null(qs[i], "Unexpected mallocx() failure, i=%u", + i); + dallocx(qs[i], MALLOCX_TCACHE(tis[i])); + } + + /* Verify that tcaches allocate cached regions. */ + for (i = 0; i < NTCACHES; i++) { + void *p0 = ps[i]; + ps[i] = mallocx(psz, MALLOCX_TCACHE(tis[i])); + assert_ptr_not_null(ps[i], "Unexpected mallocx() failure, i=%u", + i); + assert_ptr_eq(ps[i], p0, + "Expected mallocx() to allocate cached region, i=%u", i); + } + + /* Verify that reallocation uses cached regions. */ + for (i = 0; i < NTCACHES; i++) { + void *q0 = qs[i]; + qs[i] = rallocx(ps[i], qsz, MALLOCX_TCACHE(tis[i])); + assert_ptr_not_null(qs[i], "Unexpected rallocx() failure, i=%u", + i); + assert_ptr_eq(qs[i], q0, + "Expected rallocx() to allocate cached region, i=%u", i); + /* Avoid undefined behavior in case of test failure. */ + if (qs[i] == NULL) + qs[i] = ps[i]; + } + for (i = 0; i < NTCACHES; i++) + dallocx(qs[i], MALLOCX_TCACHE(tis[i])); + + /* Flush some non-empty tcaches. */ + for (i = 0; i < NTCACHES/2; i++) { + assert_d_eq(mallctl("tcache.flush", NULL, NULL, &tis[i], + sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u", + i); + } + + /* Destroy tcaches. */ + for (i = 0; i < NTCACHES; i++) { + assert_d_eq(mallctl("tcache.destroy", NULL, NULL, &tis[i], + sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u", + i); + } +} +TEST_END + TEST_BEGIN(test_thread_arena) { unsigned arena_old, arena_new, narenas; @@ -431,6 +539,8 @@ main(void) test_mallctl_config, test_mallctl_opt, test_manpage_example, + test_tcache_none, + test_tcache, test_thread_arena, test_arena_i_purge, test_arena_i_dss, From 9e561e8d3f3c625b98b57df069eeac0fa2f522fb Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 10 Feb 2015 09:03:48 -0800 Subject: [PATCH 217/352] Test and fix tcache ID recycling. --- src/tcache.c | 2 +- test/unit/mallctl.c | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/tcache.c b/src/tcache.c index c7d4f784..9fe78c39 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -451,7 +451,7 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) elm = tcaches_avail; tcaches_avail = tcaches_avail->next; elm->tcache = tcache; - *r_ind = (elm - tcaches) / sizeof(tcaches_t); + *r_ind = elm - tcaches; } else { elm = &tcaches[tcaches_past]; elm->tcache = tcache; diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 10a6fcd6..5960496f 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -258,6 +258,18 @@ TEST_BEGIN(test_tcache) "Unexpected mallctl() failure, i=%u", i); } + /* Exercise tcache ID recycling. */ + for (i = 0; i < NTCACHES; i++) { + assert_d_eq(mallctl("tcache.destroy", NULL, NULL, &tis[i], + sizeof(unsigned)), 0, "Unexpected mallctl() failure, i=%u", + i); + } + for (i = 0; i < NTCACHES; i++) { + sz = sizeof(unsigned); + assert_d_eq(mallctl("tcache.create", &tis[i], &sz, NULL, 0), 0, + "Unexpected mallctl() failure, i=%u", i); + } + /* Flush empty tcaches. */ for (i = 0; i < NTCACHES; i++) { assert_d_eq(mallctl("tcache.flush", NULL, NULL, &tis[i], From 051eae8cc591dfa2955cbfa73aae79ab53620c08 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 10 Feb 2015 16:05:52 -0800 Subject: [PATCH 218/352] Remove unnecessary xchg* lock prefixes. --- include/jemalloc/internal/atomic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h index f8bd62ec..af2c6874 100644 --- a/include/jemalloc/internal/atomic.h +++ b/include/jemalloc/internal/atomic.h @@ -119,7 +119,7 @@ atomic_write_uint64(uint64_t *p, uint64_t x) { asm volatile ( - "lock; xchgq %1, %0;" + "xchgq %1, %0;" /* Lock is implied by xchgq. */ : "=m" (*p), "+r" (x) /* Outputs. */ : "m" (*p) /* Inputs. */ : "memory" /* Clobbers. */ @@ -343,7 +343,7 @@ atomic_write_uint32(uint32_t *p, uint32_t x) { asm volatile ( - "lock; xchgl %1, %0;" + "xchgl %1, %0;" /* Lock is implied by xchgl. */ : "=m" (*p), "+r" (x) /* Outputs. */ : "m" (*p) /* Inputs. */ : "memory" /* Clobbers. */ From 064dbfbaf76617643bbbe66cbcc880e7ee9ec00f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 12 Feb 2015 00:09:37 -0800 Subject: [PATCH 219/352] Fix a regression in tcache_bin_flush_small(). Fix a serious regression in tcache_bin_flush_small() that was introduced by 1cb181ed632e7573fb4eab194e4d216867222d27 (Implement explicit tcache support.). --- src/tcache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tcache.c b/src/tcache.c index 9fe78c39..1166d60f 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -103,7 +103,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); arena_t *bin_arena = chunk->arena; - arena_bin_t *bin = &arena->bins[binind]; + arena_bin_t *bin = &bin_arena->bins[binind]; if (config_prof && bin_arena == arena) { if (arena_prof_accum(arena, tcache->prof_accumbytes)) From f30e261c5b85d2900224f91c6d426a23dce94fe9 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 12 Feb 2015 00:12:44 -0800 Subject: [PATCH 220/352] Update ckh to support metadata allocation tracking. --- src/ckh.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/ckh.c b/src/ckh.c index ad075d60..da78d1b4 100644 --- a/src/ckh.c +++ b/src/ckh.c @@ -270,8 +270,8 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) ret = true; goto label_return; } - tab = (ckhc_t *)ipalloct(tsd, usize, CACHELINE, true, NULL, - NULL); + tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, + true, NULL); if (tab == NULL) { ret = true; goto label_return; @@ -283,12 +283,12 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; if (!ckh_rebuild(ckh, tab)) { - idalloc(tsd, tab); + idalloctm(tsd, tab, tcache_get(tsd, false), true); break; } /* Rebuilding failed, so back out partially rebuilt table. */ - idalloc(tsd, ckh->tab); + idalloctm(tsd, ckh->tab, tcache_get(tsd, false), true); ckh->tab = tab; ckh->lg_curbuckets = lg_prevbuckets; } @@ -314,7 +314,8 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); if (usize == 0) return; - tab = (ckhc_t *)ipalloct(tsd, usize, CACHELINE, true, NULL, NULL); + tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true, + NULL); if (tab == NULL) { /* * An OOM error isn't worth propagating, since it doesn't @@ -329,7 +330,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; if (!ckh_rebuild(ckh, tab)) { - idalloc(tsd, tab); + idalloctm(tsd, tab, tcache_get(tsd, false), true); #ifdef CKH_COUNT ckh->nshrinks++; #endif @@ -337,7 +338,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) } /* Rebuilding failed, so back out partially rebuilt table. */ - idalloc(tsd, ckh->tab); + idalloctm(tsd, ckh->tab, tcache_get(tsd, false), true); ckh->tab = tab; ckh->lg_curbuckets = lg_prevbuckets; #ifdef CKH_COUNT @@ -390,7 +391,8 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ret = true; goto label_return; } - ckh->tab = (ckhc_t *)ipalloct(tsd, usize, CACHELINE, true, NULL, NULL); + ckh->tab = (ckhc_t *)ipallocztm(tsd, usize, CACHELINE, true, NULL, true, + NULL); if (ckh->tab == NULL) { ret = true; goto label_return; @@ -419,7 +421,7 @@ ckh_delete(tsd_t *tsd, ckh_t *ckh) (unsigned long long)ckh->nrelocs); #endif - idalloc(tsd, ckh->tab); + idalloctm(tsd, ckh->tab, tcache_get(tsd, false), true); if (config_debug) memset(ckh, 0x5a, sizeof(ckh_t)); } From cbf3a6d70371d2390b8b0e76814e04cc6088002c Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 11 Feb 2015 12:24:27 -0800 Subject: [PATCH 221/352] Move centralized chunk management into arenas. Migrate all centralized data structures related to huge allocations and recyclable chunks into arena_t, so that each arena can manage huge allocations and recyclable virtual memory completely independently of other arenas. Add chunk node caching to arenas, in order to avoid contention on the base allocator. Use chunks_rtree to look up huge allocations rather than a red-black tree. Maintain a per arena unsorted list of huge allocations (which will be needed to enumerate huge allocations during arena reset). Remove the --enable-ivsalloc option, make ivsalloc() always available, and use it for size queries if --enable-debug is enabled. The only practical implications to this removal are that 1) ivsalloc() is now always available during live debugging (and the underlying radix tree is available during core-based debugging), and 2) size query validation can no longer be enabled independent of --enable-debug. Remove the stats.chunks.{current,total,high} mallctls, and replace their underlying statistics with simpler atomically updated counters used exclusively for gdump triggering. These statistics are no longer very useful because each arena manages chunks independently, and per arena statistics provide similar information. Simplify chunk synchronization code, now that base chunk allocation cannot cause recursive lock acquisition. --- INSTALL | 6 - configure.ac | 22 +- doc/jemalloc.xml.in | 35 +-- include/jemalloc/internal/arena.h | 60 +++- include/jemalloc/internal/atomic.h | 4 +- include/jemalloc/internal/base.h | 2 - include/jemalloc/internal/chunk.h | 22 +- include/jemalloc/internal/chunk_dss.h | 4 +- include/jemalloc/internal/ctl.h | 5 - include/jemalloc/internal/extent.h | 25 +- include/jemalloc/internal/huge.h | 4 - .../jemalloc/internal/jemalloc_internal.h.in | 28 +- .../internal/jemalloc_internal_defs.h.in | 6 - include/jemalloc/internal/private_symbols.txt | 12 +- include/jemalloc/internal/rtree.h | 23 +- include/jemalloc/internal/stats.h | 15 - src/arena.c | 74 ++++- src/base.c | 65 ++--- src/chunk.c | 275 +++++++----------- src/chunk_dss.c | 5 +- src/ctl.c | 26 +- src/huge.c | 169 ++++++----- src/jemalloc.c | 15 +- src/stats.c | 12 - src/tcache.c | 8 +- test/unit/stats.c | 27 -- 26 files changed, 394 insertions(+), 555 deletions(-) diff --git a/INSTALL b/INSTALL index b8459a81..517fe021 100644 --- a/INSTALL +++ b/INSTALL @@ -92,7 +92,6 @@ any of the following arguments (not a definitive list) to 'configure': --enable-debug Enable assertions and validation code. This incurs a substantial performance hit, but is very useful during application development. - Implies --enable-ivsalloc. --enable-code-coverage Enable code coverage support, for use during jemalloc test development. @@ -107,11 +106,6 @@ any of the following arguments (not a definitive list) to 'configure': there are interactions between the various coverage targets, so it is usually advisable to run 'make clean' between repeated code coverage runs. ---enable-ivsalloc - Enable validation code, which verifies that pointers reside within - jemalloc-owned chunks before dereferencing them. This incurs a substantial - performance hit. - --disable-stats Disable statistics gathering functionality. See the "opt.stats_print" option documentation for usage details. diff --git a/configure.ac b/configure.ac index dc8aa02c..2922880a 100644 --- a/configure.ac +++ b/configure.ac @@ -625,7 +625,7 @@ fi dnl Do not compile with debugging by default. AC_ARG_ENABLE([debug], - [AS_HELP_STRING([--enable-debug], [Build debugging code (implies --enable-ivsalloc)])], + [AS_HELP_STRING([--enable-debug], [Build debugging code])], [if test "x$enable_debug" = "xno" ; then enable_debug="0" else @@ -634,27 +634,8 @@ fi ], [enable_debug="0"] ) -if test "x$enable_debug" = "x1" ; then - AC_DEFINE([JEMALLOC_DEBUG], [ ]) - enable_ivsalloc="1" -fi AC_SUBST([enable_debug]) -dnl Do not validate pointers by default. -AC_ARG_ENABLE([ivsalloc], - [AS_HELP_STRING([--enable-ivsalloc], [Validate pointers passed through the public API])], -[if test "x$enable_ivsalloc" = "xno" ; then - enable_ivsalloc="0" -else - enable_ivsalloc="1" -fi -], -[enable_ivsalloc="0"] -) -if test "x$enable_ivsalloc" = "x1" ; then - AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) -fi - dnl Only optimize if not debugging. if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS. @@ -1401,7 +1382,6 @@ if test "x${enable_zone_allocator}" = "x1" ; then if test "x${abi}" != "xmacho"; then AC_MSG_ERROR([--enable-zone-allocator is only supported on Darwin]) fi - AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) AC_DEFINE([JEMALLOC_ZONE], [ ]) dnl The szone version jumped from 3 to 6 between the OS X 10.5.x and 10.6 diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index da800ded..b392fa9e 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1847,7 +1847,7 @@ malloc_conf = "xmalloc:true";]]> equal to stats.allocated. This does not include - stats.arenas.<i>.pdirty and pages + stats.arenas.<i>.pdirty, nor pages entirely devoted to allocator metadata. @@ -1880,39 +1880,6 @@ malloc_conf = "xmalloc:true";]]> does not include inactive chunks. - - - stats.chunks.current - (size_t) - r- - [] - - Total number of chunks actively mapped on behalf of the - application. This does not include inactive chunks. - - - - - - stats.chunks.total - (uint64_t) - r- - [] - - Cumulative number of chunks allocated. - - - - - stats.chunks.high - (size_t) - r- - [] - - Maximum number of active chunks at any time thus far. - - - stats.arenas.<i>.dss diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 5476899d..2ae4609e 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -151,8 +151,12 @@ typedef ql_head(arena_chunk_map_misc_t) arena_chunk_miscelms_t; /* Arena chunk header. */ struct arena_chunk_s { - /* Arena that owns the chunk. */ - arena_t *arena; + /* + * The arena that owns the chunk is node.arena. This field as a whole + * is used by chunks_rtree to support both ivsalloc() and core-based + * debugging. + */ + extent_node_t node; /* * Map of pages within chunk that keeps track of free/large/small. The @@ -313,6 +317,27 @@ struct arena_s { /* List of dirty runs this arena manages. */ arena_chunk_miscelms_t runs_dirty; + /* Extant huge allocations. */ + ql_head(extent_node_t) huge; + /* Synchronizes all huge allocation/update/deallocation. */ + malloc_mutex_t huge_mtx; + + /* + * Trees of chunks that were previously allocated (trees differ only in + * node ordering). These are used when allocating chunks, in an attempt + * to re-use address space. Depending on function, different tree + * orderings are needed, which is why there are two trees with the same + * contents. + */ + extent_tree_t chunks_szad_mmap; + extent_tree_t chunks_ad_mmap; + extent_tree_t chunks_szad_dss; + extent_tree_t chunks_ad_dss; + malloc_mutex_t chunks_mtx; + /* Cache of nodes that were allocated via base_alloc(). */ + ql_head(extent_node_t) node_cache; + malloc_mutex_t node_cache_mtx; + /* * User-configurable chunk allocation and deallocation functions. */ @@ -338,6 +363,8 @@ extern size_t arena_maxclass; /* Max size class for arenas. */ extern unsigned nlclasses; /* Number of large size classes. */ extern unsigned nhclasses; /* Number of huge size classes. */ +extent_node_t *arena_node_alloc(arena_t *arena); +void arena_node_dalloc(arena_t *arena, extent_node_t *node); void *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, bool *zero); void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize); @@ -453,8 +480,7 @@ void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, tcache_t *tcache); arena_t *arena_aalloc(const void *ptr); size_t arena_salloc(const void *ptr, bool demote); -void arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, - tcache_t *tcache); +void arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache); void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, tcache_t *tcache); #endif @@ -792,7 +818,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) assert(binind != BININD_INVALID); assert(binind < NBINS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; + arena = chunk->node.arena; pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; actual_mapbits = arena_mapbits_get(chunk, pageind); assert(mapbits == actual_mapbits); @@ -980,7 +1006,7 @@ arena_aalloc(const void *ptr) arena_chunk_t *chunk; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - return (chunk->arena); + return (chunk->node.arena); } /* Return the size of the allocation pointed to by ptr. */ @@ -1024,11 +1050,18 @@ arena_salloc(const void *ptr, bool demote) } JEMALLOC_ALWAYS_INLINE void -arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, tcache_t *tcache) +arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) { + arena_chunk_t *chunk; size_t pageind, mapbits; assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (unlikely(chunk == ptr)) { + huge_dalloc(tsd, ptr, tcache); + return; + } assert(CHUNK_ADDR2BASE(ptr) != ptr); pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; @@ -1040,8 +1073,10 @@ arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, tcache_t *tcache) index_t binind = arena_ptr_small_binind_get(ptr, mapbits); tcache_dalloc_small(tsd, tcache, ptr, binind); - } else - arena_dalloc_small(chunk->arena, chunk, ptr, pageind); + } else { + arena_dalloc_small(chunk->node.arena, chunk, ptr, + pageind); + } } else { size_t size = arena_mapbits_large_size_get(chunk, pageind); @@ -1050,7 +1085,7 @@ arena_dalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, tcache_t *tcache) if (likely(tcache != NULL) && size <= tcache_maxclass) tcache_dalloc_large(tsd, tcache, ptr, size); else - arena_dalloc_large(chunk->arena, chunk, ptr); + arena_dalloc_large(chunk->node.arena, chunk, ptr); } } @@ -1081,7 +1116,8 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, } else { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - arena_dalloc_small(chunk->arena, chunk, ptr, pageind); + arena_dalloc_small(chunk->node.arena, chunk, ptr, + pageind); } } else { assert(((uintptr_t)ptr & PAGE_MASK) == 0); @@ -1089,7 +1125,7 @@ arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, if (likely(tcache != NULL) && size <= tcache_maxclass) tcache_dalloc_large(tsd, tcache, ptr, size); else - arena_dalloc_large(chunk->arena, chunk, ptr); + arena_dalloc_large(chunk->node.arena, chunk, ptr); } } # endif /* JEMALLOC_ARENA_INLINE_B */ diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h index af2c6874..0d33065e 100644 --- a/include/jemalloc/internal/atomic.h +++ b/include/jemalloc/internal/atomic.h @@ -52,7 +52,7 @@ void atomic_write_uint32(uint32_t *p, uint32_t x); void *atomic_add_p(void **p, void *x); void *atomic_sub_p(void **p, void *x); bool atomic_cas_p(void **p, void *c, void *s); -void atomic_write_p(void **p, void *x); +void atomic_write_p(void **p, const void *x); size_t atomic_add_z(size_t *p, size_t x); size_t atomic_sub_z(size_t *p, size_t x); bool atomic_cas_z(size_t *p, size_t c, size_t s); @@ -538,7 +538,7 @@ atomic_cas_p(void **p, void *c, void *s) } JEMALLOC_INLINE void -atomic_write_p(void **p, void *x) +atomic_write_p(void **p, const void *x) { #if (LG_SIZEOF_PTR == 3) diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index a0798ee2..bec76b32 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -10,8 +10,6 @@ #ifdef JEMALLOC_H_EXTERNS void *base_alloc(size_t size); -extent_node_t *base_node_alloc(void); -void base_node_dalloc(extent_node_t *node); size_t base_allocated_get(void); bool base_boot(void); void base_prefork(void); diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 62ac3e73..5e0fb144 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -30,24 +30,21 @@ extern size_t opt_lg_chunk; extern const char *opt_dss; -/* Protects stats_chunks; currently not used for any other purpose. */ -extern malloc_mutex_t chunks_mtx; -/* Chunk statistics. */ -extern chunk_stats_t stats_chunks; - extern rtree_t chunks_rtree; extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ extern size_t chunk_npages; +bool chunk_register(const void *chunk, const extent_node_t *node); +void chunk_deregister(const void *chunk, const extent_node_t *node); void *chunk_alloc_base(size_t size); void *chunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc, unsigned arena_ind, void *new_addr, size_t size, size_t alignment, bool *zero); void *chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind); -void chunk_unmap(void *chunk, size_t size); +void chunk_unmap(arena_t *arena, void *chunk, size_t size); bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); bool chunk_boot(void); void chunk_prefork(void); @@ -58,6 +55,19 @@ void chunk_postfork_child(void); /******************************************************************************/ #ifdef JEMALLOC_H_INLINES +#ifndef JEMALLOC_ENABLE_INLINE +extent_node_t *chunk_lookup(const void *chunk); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_CHUNK_C_)) +JEMALLOC_INLINE extent_node_t * +chunk_lookup(const void *chunk) +{ + + return (rtree_get(&chunks_rtree, (uintptr_t)chunk)); +} +#endif + #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/include/jemalloc/internal/chunk_dss.h b/include/jemalloc/internal/chunk_dss.h index 09896470..87366a28 100644 --- a/include/jemalloc/internal/chunk_dss.h +++ b/include/jemalloc/internal/chunk_dss.h @@ -23,8 +23,8 @@ extern const char *dss_prec_names[]; dss_prec_t chunk_dss_prec_get(void); bool chunk_dss_prec_set(dss_prec_t dss_prec); -void *chunk_alloc_dss(void *new_addr, size_t size, size_t alignment, - bool *zero); +void *chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, + size_t alignment, bool *zero); bool chunk_in_dss(void *chunk); bool chunk_dss_boot(void); void chunk_dss_prefork(void); diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index 65617bc9..ab9c9862 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -54,11 +54,6 @@ struct ctl_stats_s { size_t active; size_t metadata; size_t mapped; - struct { - size_t current; /* stats_chunks.curchunks */ - uint64_t total; /* stats_chunks.nchunks */ - size_t high; /* stats_chunks.highchunks */ - } chunks; unsigned narenas; ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ }; diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index f45940c1..fbcdcf99 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -9,21 +9,17 @@ typedef struct extent_node_s extent_node_t; /* Tree of extents. */ struct extent_node_s { - /* Linkage for the size/address-ordered tree. */ - rb_node(extent_node_t) link_szad; - - /* Linkage for the address-ordered tree. */ - rb_node(extent_node_t) link_ad; + /* Arena from which this extent came, if any. */ + arena_t *arena; /* Pointer to the extent that this tree node is responsible for. */ void *addr; - /* Total region size. */ + /* + * Total region size, or 0 if this node corresponds to an arena chunk. + */ size_t size; - /* Arena from which this extent came, if any. */ - arena_t *arena; - /* * 'prof_tctx' and 'zeroed' are never needed at the same time, so * overlay them in order to fit extent_node_t in one cache line. @@ -35,6 +31,17 @@ struct extent_node_s { /* True if zero-filled; used by chunk recycling code. */ bool zeroed; }; + + union { + /* Linkage for the size/address-ordered tree. */ + rb_node(extent_node_t) link_szad; + + /* Linkage for huge allocations and cached chunks nodes. */ + ql_elm(extent_node_t) link_ql; + }; + + /* Linkage for the address-ordered tree. */ + rb_node(extent_node_t) link_ad; }; typedef rb_tree(extent_node_t) extent_tree_t; diff --git a/include/jemalloc/internal/huge.h b/include/jemalloc/internal/huge.h index 231cc368..c478d16a 100644 --- a/include/jemalloc/internal/huge.h +++ b/include/jemalloc/internal/huge.h @@ -27,10 +27,6 @@ arena_t *huge_aalloc(const void *ptr); size_t huge_salloc(const void *ptr); prof_tctx_t *huge_prof_tctx_get(const void *ptr); void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx); -bool huge_boot(void); -void huge_prefork(void); -void huge_postfork_parent(void); -void huge_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index b8c994cb..ab93aa52 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -119,13 +119,6 @@ static const bool config_xmalloc = false #endif ; -static const bool config_ivsalloc = -#ifdef JEMALLOC_IVSALLOC - true -#else - false -#endif - ; #ifdef JEMALLOC_C11ATOMICS #include @@ -352,9 +345,9 @@ typedef unsigned index_t; #include "jemalloc/internal/arena.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/base.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" -#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" #include "jemalloc/internal/quarantine.h" @@ -378,9 +371,9 @@ typedef unsigned index_t; #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" #include "jemalloc/internal/base.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" -#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" #include "jemalloc/internal/quarantine.h" @@ -457,9 +450,9 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" #include "jemalloc/internal/base.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" -#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" #include "jemalloc/internal/quarantine.h" @@ -483,6 +476,7 @@ void jemalloc_postfork_child(void); #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" +#include "jemalloc/internal/rtree.h" #include "jemalloc/internal/chunk.h" #include "jemalloc/internal/huge.h" @@ -777,7 +771,6 @@ arena_get(tsd_t *tsd, unsigned ind, bool init_if_missing, #endif #include "jemalloc/internal/bitmap.h" -#include "jemalloc/internal/rtree.h" /* * Include portions of arena.h interleaved with tcache.h in order to resolve * circular dependencies. @@ -966,10 +959,14 @@ ipalloc(tsd_t *tsd, size_t usize, size_t alignment, bool zero) JEMALLOC_ALWAYS_INLINE size_t ivsalloc(const void *ptr, bool demote) { + extent_node_t *node; /* Return 0 if ptr is not within a chunk managed by jemalloc. */ - if (rtree_get(&chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0) + node = chunk_lookup(CHUNK_ADDR2BASE(ptr)); + if (node == NULL) return (0); + /* Only arena chunks should be looked up via interior pointers. */ + assert(node->addr == ptr || node->size == 0); return (isalloc(ptr, demote)); } @@ -999,7 +996,6 @@ p2rz(const void *ptr) JEMALLOC_ALWAYS_INLINE void idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata) { - arena_chunk_t *chunk; assert(ptr != NULL); if (config_stats && is_metadata) { @@ -1007,11 +1003,7 @@ idalloctm(tsd_t *tsd, void *ptr, tcache_t *tcache, bool is_metadata) config_prof)); } - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - arena_dalloc(tsd, chunk, ptr, tcache); - else - huge_dalloc(tsd, ptr, tcache); + arena_dalloc(tsd, ptr, tcache); } JEMALLOC_ALWAYS_INLINE void diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index c8d7dafb..0f0db8a1 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -186,12 +186,6 @@ #undef JEMALLOC_INTERNAL_FFSL #undef JEMALLOC_INTERNAL_FFS -/* - * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside - * within jemalloc-owned chunks before dereferencing them. - */ -#undef JEMALLOC_IVSALLOC - /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index cf42bead..d5601a68 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -60,6 +60,8 @@ arena_miscelm_to_pageind arena_miscelm_to_rpages arena_nbound arena_new +arena_node_alloc +arena_node_dalloc arena_palloc arena_postfork_child arena_postfork_parent @@ -103,8 +105,6 @@ atomic_sub_z base_alloc base_allocated_get base_boot -base_node_alloc -base_node_dalloc base_postfork_child base_postfork_parent base_prefork @@ -130,6 +130,7 @@ chunk_alloc_mmap chunk_boot chunk_dalloc_default chunk_dalloc_mmap +chunk_deregister chunk_dss_boot chunk_dss_postfork_child chunk_dss_postfork_parent @@ -137,12 +138,13 @@ chunk_dss_prec_get chunk_dss_prec_set chunk_dss_prefork chunk_in_dss +chunk_lookup chunk_npages chunk_postfork_child chunk_postfork_parent chunk_prefork +chunk_register chunk_unmap -chunks_mtx chunks_rtree chunksize chunksize_mask @@ -218,16 +220,12 @@ hash_x86_128 hash_x86_32 huge_aalloc huge_allocated -huge_boot huge_dalloc huge_dalloc_junk huge_malloc huge_ndalloc huge_nmalloc huge_palloc -huge_postfork_child -huge_postfork_parent -huge_prefork huge_prof_tctx_get huge_prof_tctx_set huge_ralloc diff --git a/include/jemalloc/internal/rtree.h b/include/jemalloc/internal/rtree.h index e86e17c4..2eb726d6 100644 --- a/include/jemalloc/internal/rtree.h +++ b/include/jemalloc/internal/rtree.h @@ -37,7 +37,7 @@ typedef void (rtree_node_dalloc_t)(rtree_node_elm_t *); struct rtree_node_elm_s { union { rtree_node_elm_t *child; - void *val; + extent_node_t *val; }; }; @@ -110,13 +110,14 @@ bool rtree_node_valid(rtree_node_elm_t *node); rtree_node_elm_t *rtree_child_tryread(rtree_node_elm_t *elm); rtree_node_elm_t *rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level); -void *rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm); -void rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, void *val); +extent_node_t *rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm); +void rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, + const extent_node_t *val); rtree_node_elm_t *rtree_subtree_tryread(rtree_t *rtree, unsigned level); rtree_node_elm_t *rtree_subtree_read(rtree_t *rtree, unsigned level); -void *rtree_get(rtree_t *rtree, uintptr_t key); -bool rtree_set(rtree_t *rtree, uintptr_t key, void *val); +extent_node_t *rtree_get(rtree_t *rtree, uintptr_t key); +bool rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_)) @@ -173,18 +174,18 @@ rtree_child_read(rtree_t *rtree, rtree_node_elm_t *elm, unsigned level) return (child); } -JEMALLOC_INLINE void * +JEMALLOC_INLINE extent_node_t * rtree_val_read(rtree_t *rtree, rtree_node_elm_t *elm) { - return (atomic_read_p(&elm->val)); + return (atomic_read_p((void **)&elm->val)); } JEMALLOC_INLINE void -rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, void *val) +rtree_val_write(rtree_t *rtree, rtree_node_elm_t *elm, const extent_node_t *val) { - atomic_write_p(&elm->val, val); + atomic_write_p((void **)&elm->val, val); } JEMALLOC_INLINE rtree_node_elm_t * @@ -210,7 +211,7 @@ rtree_subtree_read(rtree_t *rtree, unsigned level) return (subtree); } -JEMALLOC_INLINE void * +JEMALLOC_INLINE extent_node_t * rtree_get(rtree_t *rtree, uintptr_t key) { uintptr_t subkey; @@ -238,7 +239,7 @@ rtree_get(rtree_t *rtree, uintptr_t key) } JEMALLOC_INLINE bool -rtree_set(rtree_t *rtree, uintptr_t key, void *val) +rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val) { uintptr_t subkey; unsigned i, start_level; diff --git a/include/jemalloc/internal/stats.h b/include/jemalloc/internal/stats.h index 7cba77b9..c91dba99 100644 --- a/include/jemalloc/internal/stats.h +++ b/include/jemalloc/internal/stats.h @@ -135,21 +135,6 @@ struct arena_stats_s { malloc_huge_stats_t *hstats; }; -struct chunk_stats_s { - /* Number of chunks that were allocated. */ - uint64_t nchunks; - - /* High-water mark for number of chunks allocated. */ - size_t highchunks; - - /* - * Current number of chunks allocated. This value isn't maintained for - * any other purpose, so keep track of it in order to be able to set - * highchunks. - */ - size_t curchunks; -}; - #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS diff --git a/src/arena.c b/src/arena.c index 907fbd7f..2bd1a2c0 100644 --- a/src/arena.c +++ b/src/arena.c @@ -20,6 +20,7 @@ unsigned nhclasses; /* Number of huge size classes. */ * definition. */ +static void arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk); static void arena_purge(arena_t *arena, bool all); static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned); @@ -392,8 +393,7 @@ arena_chunk_init_spare(arena_t *arena) } static arena_chunk_t * -arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, - bool *zero) +arena_chunk_alloc_internal(arena_t *arena, bool *zero) { arena_chunk_t *chunk; chunk_alloc_t *chunk_alloc; @@ -403,7 +403,16 @@ arena_chunk_alloc_internal(arena_t *arena, size_t size, size_t alignment, chunk_dalloc = arena->chunk_dalloc; malloc_mutex_unlock(&arena->lock); chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, - arena->ind, NULL, size, alignment, zero); + arena->ind, NULL, chunksize, chunksize, zero); + if (chunk != NULL) { + chunk->node.arena = arena; + chunk->node.addr = chunk; + chunk->node.size = 0; /* Indicates this is an arena chunk. */ + if (chunk_register(chunk, &chunk->node)) { + chunk_dalloc((void *)chunk, chunksize, arena->ind); + chunk = NULL; + } + } malloc_mutex_lock(&arena->lock); if (config_stats && chunk != NULL) { arena->stats.mapped += chunksize; @@ -423,12 +432,10 @@ arena_chunk_init_hard(arena_t *arena) assert(arena->spare == NULL); zero = false; - chunk = arena_chunk_alloc_internal(arena, chunksize, chunksize, &zero); + chunk = arena_chunk_alloc_internal(arena, &zero); if (chunk == NULL) return (NULL); - chunk->arena = arena; - /* * Initialize the map to contain one maximal free untouched run. Mark * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. @@ -514,6 +521,7 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) } chunk_dalloc = arena->chunk_dalloc; malloc_mutex_unlock(&arena->lock); + chunk_deregister(spare, &spare->node); chunk_dalloc((void *)spare, chunksize, arena->ind); malloc_mutex_lock(&arena->lock); if (config_stats) { @@ -593,6 +601,32 @@ arena_huge_ralloc_stats_update_undo(arena_t *arena, size_t oldsize, arena_huge_malloc_stats_update_undo(arena, usize); } +extent_node_t * +arena_node_alloc(arena_t *arena) +{ + extent_node_t *node; + + malloc_mutex_lock(&arena->node_cache_mtx); + node = ql_last(&arena->node_cache, link_ql); + if (node == NULL) { + malloc_mutex_unlock(&arena->node_cache_mtx); + return (base_alloc(sizeof(extent_node_t))); + } + ql_tail_remove(&arena->node_cache, extent_node_t, link_ql); + malloc_mutex_unlock(&arena->node_cache_mtx); + return (node); +} + +void +arena_node_dalloc(arena_t *arena, extent_node_t *node) +{ + + malloc_mutex_lock(&arena->node_cache_mtx); + ql_elm_new(node, link_ql); + ql_tail_insert(&arena->node_cache, node, link_ql); + malloc_mutex_unlock(&arena->node_cache_mtx); +} + void * arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, bool *zero) @@ -1782,7 +1816,7 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, if (run == bin->runcur) bin->runcur = NULL; else { - index_t binind = arena_bin_index(chunk->arena, bin); + index_t binind = arena_bin_index(chunk->node.arena, bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; if (bin_info->nregs != 1) { @@ -2123,7 +2157,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, arena_t *arena; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; + arena = chunk->node.arena; if (usize < oldsize) { /* Fill before shrinking in order avoid a race. */ @@ -2338,10 +2372,21 @@ arena_new(unsigned ind) arena->ind = ind; arena->nthreads = 0; + if (malloc_mutex_init(&arena->lock)) + return (NULL); arena->chunk_alloc = chunk_alloc_default; arena->chunk_dalloc = chunk_dalloc_default; - - if (malloc_mutex_init(&arena->lock)) + ql_new(&arena->huge); + if (malloc_mutex_init(&arena->huge_mtx)) + return (NULL); + extent_tree_szad_new(&arena->chunks_szad_mmap); + extent_tree_ad_new(&arena->chunks_ad_mmap); + extent_tree_szad_new(&arena->chunks_szad_dss); + extent_tree_ad_new(&arena->chunks_ad_dss); + ql_new(&arena->node_cache); + if (malloc_mutex_init(&arena->chunks_mtx)) + return (NULL); + if (malloc_mutex_init(&arena->node_cache_mtx)) return (NULL); if (config_stats) { @@ -2551,6 +2596,9 @@ arena_prefork(arena_t *arena) unsigned i; malloc_mutex_prefork(&arena->lock); + malloc_mutex_prefork(&arena->huge_mtx); + malloc_mutex_prefork(&arena->chunks_mtx); + malloc_mutex_prefork(&arena->node_cache_mtx); for (i = 0; i < NBINS; i++) malloc_mutex_prefork(&arena->bins[i].lock); } @@ -2562,6 +2610,9 @@ arena_postfork_parent(arena_t *arena) for (i = 0; i < NBINS; i++) malloc_mutex_postfork_parent(&arena->bins[i].lock); + malloc_mutex_postfork_parent(&arena->node_cache_mtx); + malloc_mutex_postfork_parent(&arena->chunks_mtx); + malloc_mutex_postfork_parent(&arena->huge_mtx); malloc_mutex_postfork_parent(&arena->lock); } @@ -2572,5 +2623,8 @@ arena_postfork_child(arena_t *arena) for (i = 0; i < NBINS; i++) malloc_mutex_postfork_child(&arena->bins[i].lock); + malloc_mutex_postfork_child(&arena->node_cache_mtx); + malloc_mutex_postfork_child(&arena->chunks_mtx); + malloc_mutex_postfork_child(&arena->huge_mtx); malloc_mutex_postfork_child(&arena->lock); } diff --git a/src/base.c b/src/base.c index 0d1de7fc..7b5804ee 100644 --- a/src/base.c +++ b/src/base.c @@ -11,8 +11,9 @@ static size_t base_allocated; /******************************************************************************/ +/* base_mtx must be held. */ static extent_node_t * -base_node_try_alloc_locked(void) +base_node_try_alloc(void) { extent_node_t *node; @@ -24,8 +25,9 @@ base_node_try_alloc_locked(void) return (node); } +/* base_mtx must be held. */ static void -base_node_dalloc_locked(extent_node_t *node) +base_node_dalloc(extent_node_t *node) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); @@ -42,14 +44,14 @@ base_chunk_alloc(size_t minsize) void *addr; assert(minsize != 0); - node = base_node_try_alloc_locked(); + node = base_node_try_alloc(); /* Allocate enough space to also carve a node out if necessary. */ nsize = (node == NULL) ? CACHELINE_CEILING(sizeof(extent_node_t)) : 0; csize = CHUNK_CEILING(minsize + nsize); addr = chunk_alloc_base(csize); if (addr == NULL) { if (node != NULL) - base_node_dalloc_locked(node); + base_node_dalloc(node); return (NULL); } if (node == NULL) { @@ -63,8 +65,13 @@ base_chunk_alloc(size_t minsize) return (node); } -static void * -base_alloc_locked(size_t size) +/* + * base_alloc() guarantees demand-zeroed memory, in order to make multi-page + * sparse data structures such as radix tree nodes efficient with respect to + * physical memory usage. + */ +void * +base_alloc(size_t size) { void *ret; size_t csize; @@ -79,6 +86,7 @@ base_alloc_locked(size_t size) key.addr = NULL; key.size = csize; + malloc_mutex_lock(&base_mtx); node = extent_tree_szad_nsearch(&base_avail_szad, &key); if (node != NULL) { /* Use existing space. */ @@ -87,8 +95,10 @@ base_alloc_locked(size_t size) /* Try to allocate more space. */ node = base_chunk_alloc(csize); } - if (node == NULL) - return (NULL); + if (node == NULL) { + ret = NULL; + goto label_return; + } ret = node->addr; if (node->size > csize) { @@ -96,50 +106,15 @@ base_alloc_locked(size_t size) node->size -= csize; extent_tree_szad_insert(&base_avail_szad, node); } else - base_node_dalloc_locked(node); + base_node_dalloc(node); if (config_stats) base_allocated += csize; JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); - return (ret); -} - -/* - * base_alloc() guarantees demand-zeroed memory, in order to make multi-page - * sparse data structures such as radix tree nodes efficient with respect to - * physical memory usage. - */ -void * -base_alloc(size_t size) -{ - void *ret; - - malloc_mutex_lock(&base_mtx); - ret = base_alloc_locked(size); +label_return: malloc_mutex_unlock(&base_mtx); return (ret); } -extent_node_t * -base_node_alloc(void) -{ - extent_node_t *ret; - - malloc_mutex_lock(&base_mtx); - if ((ret = base_node_try_alloc_locked()) == NULL) - ret = (extent_node_t *)base_alloc_locked(sizeof(extent_node_t)); - malloc_mutex_unlock(&base_mtx); - return (ret); -} - -void -base_node_dalloc(extent_node_t *node) -{ - - malloc_mutex_lock(&base_mtx); - base_node_dalloc_locked(node); - malloc_mutex_unlock(&base_mtx); -} - size_t base_allocated_get(void) { diff --git a/src/chunk.c b/src/chunk.c index 9ba0b0cf..6f705ded 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -7,19 +7,9 @@ const char *opt_dss = DSS_DEFAULT; size_t opt_lg_chunk = LG_CHUNK_DEFAULT; -malloc_mutex_t chunks_mtx; -chunk_stats_t stats_chunks; - -/* - * Trees of chunks that were previously allocated (trees differ only in node - * ordering). These are used when allocating chunks, in an attempt to re-use - * address space. Depending on function, different tree orderings are needed, - * which is why there are two trees with the same contents. - */ -static extent_tree_t chunks_szad_mmap; -static extent_tree_t chunks_ad_mmap; -static extent_tree_t chunks_szad_dss; -static extent_tree_t chunks_ad_dss; +/* Used exclusively for gdump triggering. */ +static size_t curchunks; +static size_t highchunks; rtree_t chunks_rtree; @@ -29,18 +19,51 @@ size_t chunksize_mask; /* (chunksize - 1). */ size_t chunk_npages; /******************************************************************************/ -/* - * Function prototypes for static functions that are referenced prior to - * definition. - */ -static void chunk_dalloc_core(void *chunk, size_t size); +bool +chunk_register(const void *chunk, const extent_node_t *node) +{ -/******************************************************************************/ + assert(node->addr == chunk); + + if (rtree_set(&chunks_rtree, (uintptr_t)chunk, node)) + return (true); + if (config_prof && opt_prof) { + size_t nadd = (node->size == 0) ? 1 : node->size / chunksize; + size_t cur = atomic_add_z(&curchunks, nadd); + size_t high = atomic_read_z(&highchunks); + while (cur > high && atomic_cas_z(&highchunks, high, cur)) { + /* + * Don't refresh cur, because it may have decreased + * since this thread lost the highchunks update race. + */ + high = atomic_read_z(&highchunks); + } + if (cur > high && prof_gdump_get_unlocked()) + prof_gdump(); + } + + return (false); +} + +void +chunk_deregister(const void *chunk, const extent_node_t *node) +{ + bool err; + + err = rtree_set(&chunks_rtree, (uintptr_t)chunk, NULL); + assert(!err); + if (config_prof && opt_prof) { + size_t nsub = (node->size == 0) ? 1 : node->size / chunksize; + assert(atomic_read_z(&curchunks) >= nsub); + atomic_sub_z(&curchunks, nsub); + } +} static void * -chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, - void *new_addr, size_t size, size_t alignment, bool base, bool *zero) +chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, void *new_addr, size_t size, size_t alignment, + bool *zero) { void *ret; extent_node_t *node; @@ -50,27 +73,17 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, assert(new_addr == NULL || alignment == chunksize); - if (base) { - /* - * This function may need to call base_node_{,de}alloc(), but - * the current chunk allocation request is on behalf of the - * base allocator. Avoid deadlock (and if that weren't an - * issue, potential for infinite recursion) by returning NULL. - */ - return (NULL); - } - alloc_size = size + alignment - chunksize; /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); key.addr = new_addr; key.size = alloc_size; - malloc_mutex_lock(&chunks_mtx); + malloc_mutex_lock(&arena->chunks_mtx); node = (new_addr != NULL) ? extent_tree_ad_search(chunks_ad, &key) : extent_tree_szad_nsearch(chunks_szad, &key); if (node == NULL) { - malloc_mutex_unlock(&chunks_mtx); + malloc_mutex_unlock(&arena->chunks_mtx); return (NULL); } leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) - @@ -95,20 +108,12 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, if (trailsize != 0) { /* Insert the trailing space as a smaller chunk. */ if (node == NULL) { - /* - * An additional node is required, but - * base_node_alloc() can cause a new base chunk to be - * allocated. Drop chunks_mtx in order to avoid - * deadlock, and if node allocation fails, deallocate - * the result before returning an error. - */ - malloc_mutex_unlock(&chunks_mtx); - node = base_node_alloc(); + node = arena_node_alloc(arena); if (node == NULL) { - chunk_dalloc_core(ret, size); + malloc_mutex_unlock(&arena->chunks_mtx); + chunk_unmap(arena, ret, size); return (NULL); } - malloc_mutex_lock(&chunks_mtx); } node->addr = (void *)((uintptr_t)(ret) + size); node->size = trailsize; @@ -117,10 +122,10 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, extent_tree_ad_insert(chunks_ad, node); node = NULL; } - malloc_mutex_unlock(&chunks_mtx); + malloc_mutex_unlock(&arena->chunks_mtx); if (node != NULL) - base_node_dalloc(node); + arena_node_dalloc(arena, node); if (*zero) { if (!zeroed) memset(ret, 0, size); @@ -137,15 +142,15 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, } static void * -chunk_alloc_core_dss(void *new_addr, size_t size, size_t alignment, bool base, - bool *zero) +chunk_alloc_core_dss(arena_t *arena, void *new_addr, size_t size, + size_t alignment, bool *zero) { void *ret; - if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, - new_addr, size, alignment, base, zero)) != NULL) + if ((ret = chunk_recycle(arena, &arena->chunks_szad_dss, + &arena->chunks_ad_dss, new_addr, size, alignment, zero)) != NULL) return (ret); - ret = chunk_alloc_dss(new_addr, size, alignment, zero); + ret = chunk_alloc_dss(arena, new_addr, size, alignment, zero); return (ret); } @@ -156,7 +161,7 @@ chunk_alloc_core_dss(void *new_addr, size_t size, size_t alignment, bool base, * them if they are returned. */ static void * -chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, +chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero, dss_prec_t dss_prec) { void *ret; @@ -168,12 +173,13 @@ chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, /* "primary" dss. */ if (have_dss && dss_prec == dss_prec_primary && (ret = - chunk_alloc_core_dss(new_addr, size, alignment, base, zero)) != + chunk_alloc_core_dss(arena, new_addr, size, alignment, zero)) != NULL) return (ret); /* mmap. */ - if (!config_munmap && (ret = chunk_recycle(&chunks_szad_mmap, - &chunks_ad_mmap, new_addr, size, alignment, base, zero)) != NULL) + if (!config_munmap && (ret = chunk_recycle(arena, + &arena->chunks_szad_mmap, &arena->chunks_ad_mmap, new_addr, size, + alignment, zero)) != NULL) return (ret); /* * Requesting an address is not implemented for chunk_alloc_mmap(), so @@ -184,7 +190,7 @@ chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, return (ret); /* "secondary" dss. */ if (have_dss && dss_prec == dss_prec_secondary && (ret = - chunk_alloc_core_dss(new_addr, size, alignment, base, zero)) != + chunk_alloc_core_dss(arena, new_addr, size, alignment, zero)) != NULL) return (ret); @@ -192,40 +198,6 @@ chunk_alloc_core(void *new_addr, size_t size, size_t alignment, bool base, return (NULL); } -static bool -chunk_register(void *chunk, size_t size, bool base) -{ - - assert(chunk != NULL); - assert(CHUNK_ADDR2BASE(chunk) == chunk); - - if (config_ivsalloc && !base) { - if (rtree_set(&chunks_rtree, (uintptr_t)chunk, chunk)) - return (true); - } - if (config_stats || config_prof) { - bool gdump; - malloc_mutex_lock(&chunks_mtx); - if (config_stats) - stats_chunks.nchunks += (size / chunksize); - stats_chunks.curchunks += (size / chunksize); - if (stats_chunks.curchunks > stats_chunks.highchunks) { - stats_chunks.highchunks = - stats_chunks.curchunks; - if (config_prof) - gdump = true; - } else if (config_prof) - gdump = false; - malloc_mutex_unlock(&chunks_mtx); - if (config_prof && opt_prof && prof_gdump_get_unlocked() && - gdump) - prof_gdump(); - } - if (config_valgrind) - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(chunk, size); - return (false); -} - void * chunk_alloc_base(size_t size) { @@ -239,10 +211,10 @@ chunk_alloc_base(size_t size) */ zero = true; ret = chunk_alloc_mmap(size, chunksize, &zero); - if (ret != NULL && chunk_register(ret, size, true)) { - chunk_dalloc_core(ret, size); - ret = NULL; - } + if (ret == NULL) + return (NULL); + if (config_valgrind) + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); return (ret); } @@ -255,18 +227,16 @@ chunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc, void *ret; ret = chunk_alloc(new_addr, size, alignment, zero, arena_ind); - if (ret != NULL && chunk_register(ret, size, false)) { - chunk_dalloc(ret, size, arena_ind); - ret = NULL; - } + if (ret == NULL) + return (NULL); + if (config_valgrind) + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); return (ret); } -/* Default arena chunk allocation routine in the absence of user override. */ -void * -chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, - unsigned arena_ind) +static arena_t * +chunk_arena_get(unsigned arena_ind) { arena_t *arena; @@ -278,32 +248,32 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, * already. */ assert(arena != NULL); + return (arena); +} - return (chunk_alloc_core(new_addr, size, alignment, false, zero, +/* Default arena chunk allocation routine in the absence of user override. */ +void * +chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, + unsigned arena_ind) +{ + arena_t *arena; + + arena = chunk_arena_get(arena_ind); + return (chunk_alloc_core(arena, new_addr, size, alignment, zero, arena->dss_prec)); } static void -chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, - size_t size) +chunk_record(arena_t *arena, extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, void *chunk, size_t size) { bool unzeroed; - extent_node_t *xnode, *node, *prev, *xprev, key; + extent_node_t *node, *prev, key; unzeroed = pages_purge(chunk, size); JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); - /* - * Allocate a node before acquiring chunks_mtx even though it might not - * be needed, because base_node_alloc() may cause a new base chunk to - * be allocated, which could cause deadlock if chunks_mtx were already - * held. - */ - xnode = base_node_alloc(); - /* Use xprev to implement conditional deferred deallocation of prev. */ - xprev = NULL; - - malloc_mutex_lock(&chunks_mtx); + malloc_mutex_lock(&arena->chunks_mtx); key.addr = (void *)((uintptr_t)chunk + size); node = extent_tree_ad_nsearch(chunks_ad, &key); /* Try to coalesce forward. */ @@ -320,17 +290,16 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, extent_tree_szad_insert(chunks_szad, node); } else { /* Coalescing forward failed, so insert a new node. */ - if (xnode == NULL) { + node = arena_node_alloc(arena); + if (node == NULL) { /* - * base_node_alloc() failed, which is an exceedingly + * Node allocation failed, which is an exceedingly * unlikely failure. Leak chunk; its pages have * already been purged, so this is only a virtual * memory leak. */ goto label_return; } - node = xnode; - xnode = NULL; /* Prevent deallocation below. */ node->addr = chunk; node->size = size; node->zeroed = !unzeroed; @@ -356,37 +325,15 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, node->zeroed = (node->zeroed && prev->zeroed); extent_tree_szad_insert(chunks_szad, node); - xprev = prev; + arena_node_dalloc(arena, prev); } label_return: - malloc_mutex_unlock(&chunks_mtx); - /* - * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to - * avoid potential deadlock. - */ - if (xnode != NULL) - base_node_dalloc(xnode); - if (xprev != NULL) - base_node_dalloc(xprev); + malloc_mutex_unlock(&arena->chunks_mtx); } void -chunk_unmap(void *chunk, size_t size) -{ - assert(chunk != NULL); - assert(CHUNK_ADDR2BASE(chunk) == chunk); - assert(size != 0); - assert((size & chunksize_mask) == 0); - - if (have_dss && chunk_in_dss(chunk)) - chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size); - else if (chunk_dalloc_mmap(chunk, size)) - chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size); -} - -static void -chunk_dalloc_core(void *chunk, size_t size) +chunk_unmap(arena_t *arena, void *chunk, size_t size) { assert(chunk != NULL); @@ -394,16 +341,13 @@ chunk_dalloc_core(void *chunk, size_t size) assert(size != 0); assert((size & chunksize_mask) == 0); - if (config_ivsalloc) - rtree_set(&chunks_rtree, (uintptr_t)chunk, NULL); - if (config_stats || config_prof) { - malloc_mutex_lock(&chunks_mtx); - assert(stats_chunks.curchunks >= (size / chunksize)); - stats_chunks.curchunks -= (size / chunksize); - malloc_mutex_unlock(&chunks_mtx); + if (have_dss && chunk_in_dss(chunk)) { + chunk_record(arena, &arena->chunks_szad_dss, + &arena->chunks_ad_dss, chunk, size); + } else if (chunk_dalloc_mmap(chunk, size)) { + chunk_record(arena, &arena->chunks_szad_mmap, + &arena->chunks_ad_mmap, chunk, size); } - - chunk_unmap(chunk, size); } /* Default arena chunk deallocation routine in the absence of user override. */ @@ -411,7 +355,7 @@ bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) { - chunk_dalloc_core(chunk, size); + chunk_unmap(chunk_arena_get(arena_ind), chunk, size); return (false); } @@ -433,21 +377,11 @@ chunk_boot(void) chunksize_mask = chunksize - 1; chunk_npages = (chunksize >> LG_PAGE); - if (malloc_mutex_init(&chunks_mtx)) - return (true); - if (config_stats || config_prof) - memset(&stats_chunks, 0, sizeof(chunk_stats_t)); if (have_dss && chunk_dss_boot()) return (true); - extent_tree_szad_new(&chunks_szad_mmap); - extent_tree_ad_new(&chunks_ad_mmap); - extent_tree_szad_new(&chunks_szad_dss); - extent_tree_ad_new(&chunks_ad_dss); - if (config_ivsalloc) { - if (rtree_new(&chunks_rtree, (ZU(1) << (LG_SIZEOF_PTR+3)) - - opt_lg_chunk, chunks_rtree_node_alloc, NULL)) - return (true); - } + if (rtree_new(&chunks_rtree, (ZU(1) << (LG_SIZEOF_PTR+3)) - + opt_lg_chunk, chunks_rtree_node_alloc, NULL)) + return (true); return (false); } @@ -456,7 +390,6 @@ void chunk_prefork(void) { - malloc_mutex_prefork(&chunks_mtx); chunk_dss_prefork(); } @@ -465,7 +398,6 @@ chunk_postfork_parent(void) { chunk_dss_postfork_parent(); - malloc_mutex_postfork_parent(&chunks_mtx); } void @@ -473,5 +405,4 @@ chunk_postfork_child(void) { chunk_dss_postfork_child(); - malloc_mutex_postfork_child(&chunks_mtx); } diff --git a/src/chunk_dss.c b/src/chunk_dss.c index edba3b23..9c3eea82 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -66,7 +66,8 @@ chunk_dss_prec_set(dss_prec_t dss_prec) } void * -chunk_alloc_dss(void *new_addr, size_t size, size_t alignment, bool *zero) +chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, + bool *zero) { void *ret; @@ -133,7 +134,7 @@ chunk_alloc_dss(void *new_addr, size_t size, size_t alignment, bool *zero) dss_max = dss_next; malloc_mutex_unlock(&dss_mtx); if (cpad_size != 0) - chunk_unmap(cpad, cpad_size); + chunk_unmap(arena, cpad, cpad_size); if (*zero) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( ret, size); diff --git a/src/ctl.c b/src/ctl.c index a2838032..cd7927fc 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -144,9 +144,6 @@ CTL_PROTO(prof_gdump) CTL_PROTO(prof_reset) CTL_PROTO(prof_interval) CTL_PROTO(lg_prof_sample) -CTL_PROTO(stats_chunks_current) -CTL_PROTO(stats_chunks_total) -CTL_PROTO(stats_chunks_high) CTL_PROTO(stats_arenas_i_small_allocated) CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_ndalloc) @@ -363,12 +360,6 @@ static const ctl_named_node_t prof_node[] = { {NAME("lg_sample"), CTL(lg_prof_sample)} }; -static const ctl_named_node_t stats_chunks_node[] = { - {NAME("current"), CTL(stats_chunks_current)}, - {NAME("total"), CTL(stats_chunks_total)}, - {NAME("high"), CTL(stats_chunks_high)} -}; - static const ctl_named_node_t stats_arenas_i_metadata_node[] = { {NAME("mapped"), CTL(stats_arenas_i_metadata_mapped)}, {NAME("allocated"), CTL(stats_arenas_i_metadata_allocated)} @@ -473,7 +464,6 @@ static const ctl_named_node_t stats_node[] = { {NAME("active"), CTL(stats_active)}, {NAME("metadata"), CTL(stats_metadata)}, {NAME("mapped"), CTL(stats_mapped)}, - {NAME("chunks"), CHILD(named, stats_chunks)}, {NAME("arenas"), CHILD(indexed, stats_arenas)} }; @@ -688,14 +678,6 @@ ctl_refresh(void) unsigned i; VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); - if (config_stats) { - malloc_mutex_lock(&chunks_mtx); - ctl_stats.chunks.current = stats_chunks.curchunks; - ctl_stats.chunks.total = stats_chunks.nchunks; - ctl_stats.chunks.high = stats_chunks.highchunks; - malloc_mutex_unlock(&chunks_mtx); - } - /* * Clear sum stats, since they will be merged into by * ctl_arena_refresh(). @@ -733,7 +715,8 @@ ctl_refresh(void) + ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped + ctl_stats.arenas[ctl_stats.narenas].astats .metadata_allocated; - ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); + ctl_stats.mapped = + ctl_stats.arenas[ctl_stats.narenas].astats.mapped; } ctl_epoch++; @@ -1950,11 +1933,6 @@ CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) -CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current, - size_t) -CTL_RO_CGEN(config_stats, stats_chunks_total, ctl_stats.chunks.total, uint64_t) -CTL_RO_CGEN(config_stats, stats_chunks_high, ctl_stats.chunks.high, size_t) - CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) diff --git a/src/huge.c b/src/huge.c index db0ecd51..00327277 100644 --- a/src/huge.c +++ b/src/huge.c @@ -2,15 +2,33 @@ #include "jemalloc/internal/jemalloc_internal.h" /******************************************************************************/ -/* Data. */ -/* Protects chunk-related data structures. */ -static malloc_mutex_t huge_mtx; +static extent_node_t * +huge_node_get(const void *ptr) +{ + extent_node_t *node; -/******************************************************************************/ + node = chunk_lookup(ptr); + assert(node->size != 0); -/* Tree of chunks that are stand-alone huge allocations. */ -static extent_tree_t huge; + return (node); +} + +static bool +huge_node_set(const void *ptr, extent_node_t *node) +{ + + assert(node->addr == ptr); + assert(node->size != 0); + return (chunk_register(ptr, node)); +} + +static void +huge_node_unset(const void *ptr, const extent_node_t *node) +{ + + chunk_deregister(ptr, node); +} void * huge_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, @@ -55,15 +73,22 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, return (NULL); } - /* Insert node into huge. */ node->addr = ret; node->size = usize; node->zeroed = is_zeroed; node->arena = arena; - malloc_mutex_lock(&huge_mtx); - extent_tree_ad_insert(&huge, node); - malloc_mutex_unlock(&huge_mtx); + if (huge_node_set(ret, node)) { + arena_chunk_dalloc_huge(arena, ret, usize); + idalloctm(tsd, node, tcache, true); + return (NULL); + } + + /* Insert node into huge. */ + malloc_mutex_lock(&arena->huge_mtx); + ql_elm_new(node, link_ql); + ql_tail_insert(&arena->huge, node, link_ql); + malloc_mutex_unlock(&arena->huge_mtx); if (zero || (config_fill && unlikely(opt_zero))) { if (!is_zeroed) @@ -74,32 +99,6 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, return (ret); } -static extent_node_t * -huge_node_locked(const void *ptr) -{ - extent_node_t *node, key; - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); - - return (node); -} - -static extent_node_t * -huge_node(const void *ptr) -{ - extent_node_t *node; - - malloc_mutex_lock(&huge_mtx); - node = huge_node_locked(ptr); - malloc_mutex_unlock(&huge_mtx); - - return (node); -} - #ifdef JEMALLOC_JET #undef huge_dalloc_junk #define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) @@ -152,15 +151,15 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, } else zeroed = true; - malloc_mutex_lock(&huge_mtx); - node = huge_node_locked(ptr); + node = huge_node_get(ptr); arena = node->arena; + malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ assert(node->size != usize); node->size = usize; /* Clear node->zeroed if zeroing failed above. */ node->zeroed = (node->zeroed && zeroed); - malloc_mutex_unlock(&huge_mtx); + malloc_mutex_unlock(&arena->huge_mtx); arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize); @@ -195,14 +194,14 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) zeroed = false; } - malloc_mutex_lock(&huge_mtx); - node = huge_node_locked(ptr); + node = huge_node_get(ptr); arena = node->arena; + malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ node->size = usize; /* Clear node->zeroed if zeroing failed above. */ node->zeroed = (node->zeroed && zeroed); - malloc_mutex_unlock(&huge_mtx); + malloc_mutex_unlock(&arena->huge_mtx); /* Zap the excess chunks. */ arena_chunk_ralloc_huge_shrink(arena, ptr, oldsize, usize); @@ -221,11 +220,11 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { return (true); } - malloc_mutex_lock(&huge_mtx); - node = huge_node_locked(ptr); + node = huge_node_get(ptr); arena = node->arena; + malloc_mutex_lock(&arena->huge_mtx); is_zeroed_subchunk = node->zeroed; - malloc_mutex_unlock(&huge_mtx); + malloc_mutex_unlock(&arena->huge_mtx); /* * Copy zero into is_zeroed_chunk and pass the copy to chunk_alloc(), so @@ -237,10 +236,10 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { &is_zeroed_chunk)) return (true); - malloc_mutex_lock(&huge_mtx); + malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ node->size = usize; - malloc_mutex_unlock(&huge_mtx); + malloc_mutex_unlock(&arena->huge_mtx); if (zero || (config_fill && unlikely(opt_zero))) { if (!is_zeroed_subchunk) { @@ -356,11 +355,14 @@ void huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) { extent_node_t *node; + arena_t *arena; - malloc_mutex_lock(&huge_mtx); - node = huge_node_locked(ptr); - extent_tree_ad_remove(&huge, node); - malloc_mutex_unlock(&huge_mtx); + node = huge_node_get(ptr); + arena = node->arena; + huge_node_unset(ptr, node); + malloc_mutex_lock(&arena->huge_mtx); + ql_remove(&arena->huge, node, link_ql); + malloc_mutex_unlock(&arena->huge_mtx); huge_dalloc_junk(node->addr, node->size); arena_chunk_dalloc_huge(node->arena, node->addr, node->size); @@ -371,59 +373,50 @@ arena_t * huge_aalloc(const void *ptr) { - return (huge_node(ptr)->arena); + return (huge_node_get(ptr)->arena); } size_t huge_salloc(const void *ptr) { + size_t size; + extent_node_t *node; + arena_t *arena; - return (huge_node(ptr)->size); + node = huge_node_get(ptr); + arena = node->arena; + malloc_mutex_lock(&arena->huge_mtx); + size = node->size; + malloc_mutex_unlock(&arena->huge_mtx); + + return (size); } prof_tctx_t * huge_prof_tctx_get(const void *ptr) { + prof_tctx_t *tctx; + extent_node_t *node; + arena_t *arena; - return (huge_node(ptr)->prof_tctx); + node = huge_node_get(ptr); + arena = node->arena; + malloc_mutex_lock(&arena->huge_mtx); + tctx = node->prof_tctx; + malloc_mutex_unlock(&arena->huge_mtx); + + return (tctx); } void huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { + extent_node_t *node; + arena_t *arena; - huge_node(ptr)->prof_tctx = tctx; -} - -bool -huge_boot(void) -{ - - /* Initialize chunks data. */ - if (malloc_mutex_init(&huge_mtx)) - return (true); - extent_tree_ad_new(&huge); - - return (false); -} - -void -huge_prefork(void) -{ - - malloc_mutex_prefork(&huge_mtx); -} - -void -huge_postfork_parent(void) -{ - - malloc_mutex_postfork_parent(&huge_mtx); -} - -void -huge_postfork_child(void) -{ - - malloc_mutex_postfork_child(&huge_mtx); + node = huge_node_get(ptr); + arena = node->arena; + malloc_mutex_lock(&arena->huge_mtx); + node->prof_tctx = tctx; + malloc_mutex_unlock(&arena->huge_mtx); } diff --git a/src/jemalloc.c b/src/jemalloc.c index 94477914..3903209b 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1195,8 +1195,6 @@ malloc_init_hard_a0_locked(void) return (true); if (config_tcache && tcache_boot()) malloc_mutex_unlock(&init_lock); - if (huge_boot()) - return (true); if (malloc_mutex_init(&arenas_lock)) return (true); /* @@ -2310,12 +2308,10 @@ je_sallocx(const void *ptr, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); - if (config_ivsalloc) + if (config_debug) usize = ivsalloc(ptr, config_prof); - else { - assert(ptr != NULL); + else usize = isalloc(ptr, config_prof); - } return (usize); } @@ -2440,10 +2436,10 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); - if (config_ivsalloc) + if (config_debug) ret = ivsalloc(ptr, config_prof); else - ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; + ret = (ptr == NULL) ? 0 : isalloc(ptr, config_prof); return (ret); } @@ -2504,7 +2500,6 @@ _malloc_prefork(void) } chunk_prefork(); base_prefork(); - huge_prefork(); } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -2524,7 +2519,6 @@ _malloc_postfork(void) assert(malloc_initialized()); /* Release all mutexes, now that fork() has completed. */ - huge_postfork_parent(); base_postfork_parent(); chunk_postfork_parent(); for (i = 0; i < narenas_total; i++) { @@ -2544,7 +2538,6 @@ jemalloc_postfork_child(void) assert(malloc_initialized()); /* Release all mutexes, now that fork() has completed. */ - huge_postfork_child(); base_postfork_child(); chunk_postfork_child(); for (i = 0; i < narenas_total; i++) { diff --git a/src/stats.c b/src/stats.c index 865f7757..e0f71651 100644 --- a/src/stats.c +++ b/src/stats.c @@ -547,8 +547,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, if (config_stats) { size_t *cactive; size_t allocated, active, metadata, mapped; - size_t chunks_current, chunks_high; - uint64_t chunks_total; CTL_GET("stats.cactive", &cactive, size_t *); CTL_GET("stats.allocated", &allocated, size_t); @@ -561,16 +559,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "Current active ceiling: %zu\n", atomic_read_z(cactive)); - /* Print chunk stats. */ - CTL_GET("stats.chunks.total", &chunks_total, uint64_t); - CTL_GET("stats.chunks.high", &chunks_high, size_t); - CTL_GET("stats.chunks.current", &chunks_current, size_t); - malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " - "highchunks curchunks\n"); - malloc_cprintf(write_cb, cbopaque, - " %13"PRIu64" %12zu %12zu\n", - chunks_total, chunks_high, chunks_current); - if (merged) { unsigned narenas; diff --git a/src/tcache.c b/src/tcache.c index 1166d60f..10c85dd3 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -102,7 +102,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, /* Lock the arena bin associated with the first object. */ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); - arena_t *bin_arena = chunk->arena; + arena_t *bin_arena = chunk->node.arena; arena_bin_t *bin = &bin_arena->bins[binind]; if (config_prof && bin_arena == arena) { @@ -124,7 +124,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->arena == bin_arena) { + if (chunk->node.arena == bin_arena) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_bits_t *bitselm = @@ -182,7 +182,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, /* Lock the arena associated with the first object. */ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); - arena_t *locked_arena = chunk->arena; + arena_t *locked_arena = chunk->node.arena; UNUSED bool idump; if (config_prof) @@ -208,7 +208,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->arena == locked_arena) { + if (chunk->node.arena == locked_arena) { arena_dalloc_large_junked_locked(locked_arena, chunk, ptr); } else { diff --git a/test/unit/stats.c b/test/unit/stats.c index 946e7370..10999670 100644 --- a/test/unit/stats.c +++ b/test/unit/stats.c @@ -29,32 +29,6 @@ TEST_BEGIN(test_stats_summary) } TEST_END -TEST_BEGIN(test_stats_chunks) -{ - size_t current, high; - uint64_t total; - size_t sz; - int expected = config_stats ? 0 : ENOENT; - - sz = sizeof(size_t); - assert_d_eq(mallctl("stats.chunks.current", ¤t, &sz, NULL, 0), - expected, "Unexpected mallctl() result"); - sz = sizeof(uint64_t); - assert_d_eq(mallctl("stats.chunks.total", &total, &sz, NULL, 0), - expected, "Unexpected mallctl() result"); - sz = sizeof(size_t); - assert_d_eq(mallctl("stats.chunks.high", &high, &sz, NULL, 0), expected, - "Unexpected mallctl() result"); - - if (config_stats) { - assert_zu_le(current, high, - "current should be no larger than high"); - assert_u64_le((uint64_t)high, total, - "high should be no larger than total"); - } -} -TEST_END - TEST_BEGIN(test_stats_huge) { void *p; @@ -458,7 +432,6 @@ main(void) return (test( test_stats_summary, - test_stats_chunks, test_stats_huge, test_stats_arenas_summary, test_stats_arenas_small, From 1eaf3b6f345e0b5835549f19e844c81314c90435 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Thu, 12 Feb 2015 15:46:30 -0500 Subject: [PATCH 222/352] add missing check for new_addr chunk size 8ddc93293cd8370870f221225ef1e013fbff6d65 switched this to over using the address tree in order to avoid false negatives, so it now needs to check that the size of the free extent is large enough to satisfy the request. --- src/chunk.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chunk.c b/src/chunk.c index 6f705ded..b3576195 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -82,7 +82,7 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, malloc_mutex_lock(&arena->chunks_mtx); node = (new_addr != NULL) ? extent_tree_ad_search(chunks_ad, &key) : extent_tree_szad_nsearch(chunks_szad, &key); - if (node == NULL) { + if (node == NULL || (new_addr != NULL && node->size < size)) { malloc_mutex_unlock(&arena->chunks_mtx); return (NULL); } From 88fef7ceda6269598cef0cee8b984c8765673c27 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 12 Feb 2015 14:06:37 -0800 Subject: [PATCH 223/352] Refactor huge_*() calls into arena internals. Make redirects to the huge_*() API the arena code's responsibility, since arenas now take responsibility for all allocation sizes. --- include/jemalloc/internal/arena.h | 222 ++++++++++-------- .../jemalloc/internal/jemalloc_internal.h.in | 64 +---- include/jemalloc/internal/prof.h | 17 +- src/arena.c | 162 ++++++++----- 4 files changed, 238 insertions(+), 227 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 2ae4609e..77a7dcb6 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -391,7 +391,8 @@ void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); void arena_quarantine_junk_small(void *ptr, size_t usize); void *arena_malloc_small(arena_t *arena, size_t size, bool zero); void *arena_malloc_large(arena_t *arena, size_t size, bool zero); -void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); +void *arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, + size_t alignment, bool zero, tcache_t *tcache); void arena_prof_promoted(const void *ptr, size_t size); void arena_dalloc_bin_junked_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_bits_t *bitselm); @@ -481,8 +482,7 @@ void *arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, arena_t *arena_aalloc(const void *ptr); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache); -void arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, - tcache_t *tcache); +void arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) @@ -931,20 +931,22 @@ arena_prof_tctx_get(const void *ptr) { prof_tctx_t *ret; arena_chunk_t *chunk; - size_t pageind, mapbits; cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) - ret = (prof_tctx_t *)(uintptr_t)1U; - else - ret = arena_miscelm_get(chunk, pageind)->prof_tctx; + if (likely(chunk != ptr)) { + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t mapbits = arena_mapbits_get(chunk, pageind); + assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); + if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) + ret = (prof_tctx_t *)(uintptr_t)1U; + else + ret = arena_miscelm_get(chunk, pageind)->prof_tctx; + } else + ret = huge_prof_tctx_get(ptr); return (ret); } @@ -953,18 +955,20 @@ JEMALLOC_INLINE void arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { arena_chunk_t *chunk; - size_t pageind; cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + if (likely(chunk != ptr)) { + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) - arena_miscelm_get(chunk, pageind)->prof_tctx = tctx; + if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) + arena_miscelm_get(chunk, pageind)->prof_tctx = tctx; + } else + huge_prof_tctx_set(ptr, tctx); } JEMALLOC_ALWAYS_INLINE void * @@ -984,7 +988,7 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, return (NULL); return (arena_malloc_small(arena, size, zero)); } - } else { + } else if (likely(size <= arena_maxclass)) { /* * Initialize tcache after checking size in order to avoid * infinite recursion during tcache initialization. @@ -997,7 +1001,8 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, return (NULL); return (arena_malloc_large(arena, size, zero)); } - } + } else + return (huge_malloc(tsd, arena, size, zero, tcache)); } JEMALLOC_ALWAYS_INLINE arena_t * @@ -1006,7 +1011,10 @@ arena_aalloc(const void *ptr) arena_chunk_t *chunk; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - return (chunk->node.arena); + if (likely(chunk != ptr)) + return (chunk->node.arena); + else + return (huge_aalloc(ptr)); } /* Return the size of the allocation pointed to by ptr. */ @@ -1022,29 +1030,37 @@ arena_salloc(const void *ptr, bool demote) assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - binind = arena_mapbits_binind_get(chunk, pageind); - if (unlikely(binind == BININD_INVALID || (config_prof && !demote && - arena_mapbits_large_get(chunk, pageind) != 0))) { - /* - * Large allocation. In the common case (demote), and as this - * is an inline function, most callers will only end up looking - * at binind to determine that ptr is a small allocation. - */ - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = arena_mapbits_large_size_get(chunk, pageind); - assert(ret != 0); - assert(pageind + (ret>>LG_PAGE) <= chunk_npages); - assert(arena_mapbits_dirty_get(chunk, pageind) == - arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); - } else { - /* Small allocation (possibly promoted to a large object). */ - assert(arena_mapbits_large_get(chunk, pageind) != 0 || - arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, - pageind)) == binind); - ret = index2size(binind); - } + if (likely(chunk != ptr)) { + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + binind = arena_mapbits_binind_get(chunk, pageind); + if (unlikely(binind == BININD_INVALID || (config_prof && !demote + && arena_mapbits_large_get(chunk, pageind) != 0))) { + /* + * Large allocation. In the common case (demote), and + * as this is an inline function, most callers will only + * end up looking at binind to determine that ptr is a + * small allocation. + */ + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + ret = arena_mapbits_large_size_get(chunk, pageind); + assert(ret != 0); + assert(pageind + (ret>>LG_PAGE) <= chunk_npages); + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, + pageind+(ret>>LG_PAGE)-1)); + } else { + /* + * Small allocation (possibly promoted to a large + * object). + */ + assert(arena_mapbits_large_get(chunk, pageind) != 0 || + arena_ptr_small_binind_get(ptr, + arena_mapbits_get(chunk, pageind)) == binind); + ret = index2size(binind); + } + } else + ret = huge_salloc(ptr); return (ret); } @@ -1058,75 +1074,83 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (unlikely(chunk == ptr)) { - huge_dalloc(tsd, ptr, tcache); - return; - } - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - mapbits = arena_mapbits_get(chunk, pageind); - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { - /* Small allocation. */ - if (likely(tcache != NULL)) { - index_t binind = arena_ptr_small_binind_get(ptr, - mapbits); - tcache_dalloc_small(tsd, tcache, ptr, binind); + if (likely(chunk != ptr)) { + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + mapbits = arena_mapbits_get(chunk, pageind); + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) { + /* Small allocation. */ + if (likely(tcache != NULL)) { + index_t binind = arena_ptr_small_binind_get(ptr, + mapbits); + tcache_dalloc_small(tsd, tcache, ptr, binind); + } else { + arena_dalloc_small(chunk->node.arena, chunk, + ptr, pageind); + } } else { - arena_dalloc_small(chunk->node.arena, chunk, ptr, + size_t size = arena_mapbits_large_size_get(chunk, pageind); + + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + + if (likely(tcache != NULL) && size <= tcache_maxclass) + tcache_dalloc_large(tsd, tcache, ptr, size); + else { + arena_dalloc_large(chunk->node.arena, chunk, + ptr); + } } - } else { - size_t size = arena_mapbits_large_size_get(chunk, pageind); - - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - - if (likely(tcache != NULL) && size <= tcache_maxclass) - tcache_dalloc_large(tsd, tcache, ptr, size); - else - arena_dalloc_large(chunk->node.arena, chunk, ptr); - } + } else + huge_dalloc(tsd, ptr, tcache); } JEMALLOC_ALWAYS_INLINE void -arena_sdalloc(tsd_t *tsd, arena_chunk_t *chunk, void *ptr, size_t size, - tcache_t *tcache) +arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) { + arena_chunk_t *chunk; - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - if (config_prof && opt_prof) { - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (arena_mapbits_large_get(chunk, pageind) != 0) { - /* Make sure to use promoted size, not request size. */ - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - size = arena_mapbits_large_size_get(chunk, pageind); - } - } - assert(s2u(size) == s2u(arena_salloc(ptr, false))); - - if (likely(size <= SMALL_MAXCLASS)) { - /* Small allocation. */ - if (likely(tcache != NULL)) { - index_t binind = size2index(size); - tcache_dalloc_small(tsd, tcache, ptr, binind); - } else { + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (likely(chunk != ptr)) { + if (config_prof && opt_prof) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - arena_dalloc_small(chunk->node.arena, chunk, ptr, - pageind); + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + if (arena_mapbits_large_get(chunk, pageind) != 0) { + /* + * Make sure to use promoted size, not request + * size. + */ + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + size = arena_mapbits_large_size_get(chunk, + pageind); + } } - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); + assert(s2u(size) == s2u(arena_salloc(ptr, false))); - if (likely(tcache != NULL) && size <= tcache_maxclass) - tcache_dalloc_large(tsd, tcache, ptr, size); - else - arena_dalloc_large(chunk->node.arena, chunk, ptr); - } + if (likely(size <= SMALL_MAXCLASS)) { + /* Small allocation. */ + if (likely(tcache != NULL)) { + index_t binind = size2index(size); + tcache_dalloc_small(tsd, tcache, ptr, binind); + } else { + size_t pageind = ((uintptr_t)ptr - + (uintptr_t)chunk) >> LG_PAGE; + arena_dalloc_small(chunk->node.arena, chunk, + ptr, pageind); + } + } else { + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + + if (likely(tcache != NULL) && size <= tcache_maxclass) + tcache_dalloc_large(tsd, tcache, ptr, size); + else { + arena_dalloc_large(chunk->node.arena, chunk, + ptr); + } + } + } else + huge_dalloc(tsd, ptr, tcache); } # endif /* JEMALLOC_ARENA_INLINE_B */ #endif diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index ab93aa52..43276c6c 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -823,18 +823,10 @@ bool ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, JEMALLOC_ALWAYS_INLINE arena_t * iaalloc(const void *ptr) { - arena_t *arena; - arena_chunk_t *chunk; assert(ptr != NULL); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - arena = arena_aalloc(ptr); - else - arena = huge_aalloc(ptr); - - return (arena); + return (arena_aalloc(ptr)); } /* @@ -845,20 +837,12 @@ iaalloc(const void *ptr) JEMALLOC_ALWAYS_INLINE size_t isalloc(const void *ptr, bool demote) { - size_t ret; - arena_chunk_t *chunk; assert(ptr != NULL); /* Demotion only makes sense if config_prof is true. */ assert(config_prof || !demote); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - ret = arena_salloc(ptr, demote); - else - ret = huge_salloc(ptr); - - return (ret); + return (arena_salloc(ptr, demote)); } JEMALLOC_ALWAYS_INLINE void * @@ -869,10 +853,7 @@ iallocztm(tsd_t *tsd, size_t size, bool zero, tcache_t *tcache, bool is_metadata assert(size != 0); - if (likely(size <= arena_maxclass)) - ret = arena_malloc(tsd, arena, size, zero, tcache); - else - ret = huge_malloc(tsd, arena, size, zero, tcache); + ret = arena_malloc(tsd, arena, size, zero, tcache); if (config_stats && is_metadata && likely(ret != NULL)) { arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, config_prof)); @@ -917,21 +898,7 @@ ipallocztm(tsd_t *tsd, size_t usize, size_t alignment, bool zero, assert(usize != 0); assert(usize == sa2u(usize, alignment)); - if (usize <= SMALL_MAXCLASS && alignment < PAGE) - ret = arena_malloc(tsd, arena, usize, zero, tcache); - else { - if (likely(usize <= arena_maxclass)) { - arena = arena_choose(tsd, arena); - if (unlikely(arena == NULL)) - return (NULL); - ret = arena_palloc(arena, usize, alignment, zero); - } else if (likely(alignment <= chunksize)) - ret = huge_malloc(tsd, arena, usize, zero, tcache); - else { - ret = huge_palloc(tsd, arena, usize, alignment, zero, - tcache); - } - } + ret = arena_palloc(tsd, arena, usize, alignment, zero, tcache); assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); if (config_stats && is_metadata && likely(ret != NULL)) { arena_metadata_allocated_add(iaalloc(ret), isalloc(ret, @@ -1033,15 +1000,8 @@ iqalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) JEMALLOC_ALWAYS_INLINE void isdalloct(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) { - arena_chunk_t *chunk; - assert(ptr != NULL); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - arena_sdalloc(tsd, chunk, ptr, size, tcache); - else - huge_dalloc(tsd, ptr, tcache); + arena_sdalloc(tsd, ptr, size, tcache); } JEMALLOC_ALWAYS_INLINE void @@ -1104,13 +1064,8 @@ iralloct(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, zero, tcache, arena)); } - if (likely(size <= arena_maxclass)) { - return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, - alignment, zero, tcache)); - } else { - return (huge_ralloc(tsd, arena, ptr, oldsize, size, 0, - alignment, zero, tcache)); - } + return (arena_ralloc(tsd, arena, ptr, oldsize, size, 0, alignment, zero, + tcache)); } JEMALLOC_ALWAYS_INLINE void * @@ -1136,10 +1091,7 @@ ixalloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, return (true); } - if (likely(size <= arena_maxclass)) - return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero)); - else - return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero)); + return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero)); } #endif diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index b2db6859..f5082438 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -372,34 +372,21 @@ prof_tdata_get(tsd_t *tsd, bool create) JEMALLOC_ALWAYS_INLINE prof_tctx_t * prof_tctx_get(const void *ptr) { - prof_tctx_t *ret; - arena_chunk_t *chunk; cassert(config_prof); assert(ptr != NULL); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - ret = arena_prof_tctx_get(ptr); - else - ret = huge_prof_tctx_get(ptr); - - return (ret); + return (arena_prof_tctx_get(ptr)); } JEMALLOC_ALWAYS_INLINE void prof_tctx_set(const void *ptr, prof_tctx_t *tctx) { - arena_chunk_t *chunk; cassert(config_prof); assert(ptr != NULL); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (likely(chunk != ptr)) - arena_prof_tctx_set(ptr, tctx); - else - huge_prof_tctx_set(ptr, tctx); + arena_prof_tctx_set(ptr, tctx); } JEMALLOC_ALWAYS_INLINE bool diff --git a/src/arena.c b/src/arena.c index 2bd1a2c0..7b441bed 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1714,8 +1714,9 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) } /* Only handles large allocations that require more than page alignment. */ -void * -arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) +static void * +arena_palloc_large(tsd_t *tsd, arena_t *arena, size_t size, size_t alignment, + bool zero) { void *ret; size_t alloc_size, leadsize, trailsize; @@ -1726,6 +1727,10 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) assert((size & PAGE_MASK) == 0); + arena = arena_choose(tsd, arena); + if (unlikely(arena == NULL)) + return (NULL); + alignment = PAGE_CEILING(alignment); alloc_size = size + alignment - PAGE; @@ -1783,6 +1788,28 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) return (ret); } +void * +arena_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, + bool zero, tcache_t *tcache) +{ + void *ret; + + if (usize <= SMALL_MAXCLASS && alignment < PAGE) + ret = arena_malloc(tsd, arena, usize, zero, tcache); + else { + if (likely(usize <= arena_maxclass)) { + ret = arena_palloc_large(tsd, arena, usize, alignment, + zero); + } else if (likely(alignment <= chunksize)) + ret = huge_malloc(tsd, arena, usize, zero, tcache); + else { + ret = huge_palloc(tsd, arena, usize, alignment, zero, + tcache); + } + } + return (ret); +} + void arena_prof_promoted(const void *ptr, size_t size) { @@ -2189,29 +2216,35 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { - /* - * Avoid moving the allocation if the size class can be left the same. - */ - if (likely(oldsize <= arena_maxclass)) { - if (oldsize <= SMALL_MAXCLASS) { - assert(arena_bin_info[size2index(oldsize)].reg_size - == oldsize); - if ((size + extra <= SMALL_MAXCLASS && size2index(size + - extra) == size2index(oldsize)) || (size <= oldsize - && size + extra >= oldsize)) - return (false); - } else { - assert(size <= arena_maxclass); - if (size + extra > SMALL_MAXCLASS) { - if (!arena_ralloc_large(ptr, oldsize, size, - extra, zero)) + if (likely(size <= arena_maxclass)) { + /* + * Avoid moving the allocation if the size class can be left the + * same. + */ + if (likely(oldsize <= arena_maxclass)) { + if (oldsize <= SMALL_MAXCLASS) { + assert( + arena_bin_info[size2index(oldsize)].reg_size + == oldsize); + if ((size + extra <= SMALL_MAXCLASS && + size2index(size + extra) == + size2index(oldsize)) || (size <= oldsize && + size + extra >= oldsize)) return (false); + } else { + assert(size <= arena_maxclass); + if (size + extra > SMALL_MAXCLASS) { + if (!arena_ralloc_large(ptr, oldsize, + size, extra, zero)) + return (false); + } } } - } - /* Reallocation would require a move. */ - return (true); + /* Reallocation would require a move. */ + return (true); + } else + return (huge_ralloc_no_move(ptr, oldsize, size, extra, zero)); } void * @@ -2219,52 +2252,67 @@ arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache) { void *ret; - size_t copysize; - /* Try to avoid moving the allocation. */ - if (!arena_ralloc_no_move(ptr, oldsize, size, extra, zero)) - return (ptr); + if (likely(size <= arena_maxclass)) { + size_t copysize; - /* - * size and oldsize are different enough that we need to move the - * object. In that case, fall back to allocating new space and - * copying. - */ - if (alignment != 0) { - size_t usize = sa2u(size + extra, alignment); - if (usize == 0) - return (NULL); - ret = ipalloct(tsd, usize, alignment, zero, tcache, arena); - } else - ret = arena_malloc(tsd, arena, size + extra, zero, tcache); + /* Try to avoid moving the allocation. */ + if (!arena_ralloc_no_move(ptr, oldsize, size, extra, zero)) + return (ptr); - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, this time without extra. */ + /* + * size and oldsize are different enough that we need to move + * the object. In that case, fall back to allocating new space + * and copying. + */ if (alignment != 0) { - size_t usize = sa2u(size, alignment); + size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); ret = ipalloct(tsd, usize, alignment, zero, tcache, arena); - } else - ret = arena_malloc(tsd, arena, size, zero, tcache); + } else { + ret = arena_malloc(tsd, arena, size + extra, zero, + tcache); + } - if (ret == NULL) - return (NULL); + if (ret == NULL) { + if (extra == 0) + return (NULL); + /* Try again, this time without extra. */ + if (alignment != 0) { + size_t usize = sa2u(size, alignment); + if (usize == 0) + return (NULL); + ret = ipalloct(tsd, usize, alignment, zero, + tcache, arena); + } else { + ret = arena_malloc(tsd, arena, size, zero, + tcache); + } + + if (ret == NULL) + return (NULL); + } + + /* + * Junk/zero-filling were already done by + * ipalloc()/arena_malloc(). + */ + + /* + * Copy at most size bytes (not size+extra), since the caller + * has no expectation that the extra bytes will be reliably + * preserved. + */ + copysize = (size < oldsize) ? size : oldsize; + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); + memcpy(ret, ptr, copysize); + isqalloc(tsd, ptr, oldsize, tcache); + } else { + ret = huge_ralloc(tsd, arena, ptr, oldsize, size, extra, + alignment, zero, tcache); } - - /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ - - /* - * Copy at most size bytes (not size+extra), since the caller has no - * expectation that the extra bytes will be reliably preserved. - */ - copysize = (size < oldsize) ? size : oldsize; - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); - memcpy(ret, ptr, copysize); - isqalloc(tsd, ptr, oldsize, tcache); return (ret); } From 5f7140b045136232b1bbe66fcf2a7f63d08682a1 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 12 Feb 2015 15:54:53 -0800 Subject: [PATCH 224/352] Make prof_tctx accesses atomic. Although exceedingly unlikely, it appears that writes to the prof_tctx field of arena_chunk_map_misc_t could be reordered such that a stale value could be read during deallocation, with profiler metadata corruption and invalid pointer dereferences being the most likely effects. --- include/jemalloc/internal/arena.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 77a7dcb6..4d88736d 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -943,8 +943,11 @@ arena_prof_tctx_get(const void *ptr) assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); if (likely((mapbits & CHUNK_MAP_LARGE) == 0)) ret = (prof_tctx_t *)(uintptr_t)1U; - else - ret = arena_miscelm_get(chunk, pageind)->prof_tctx; + else { + arena_chunk_map_misc_t *elm = arena_miscelm_get(chunk, + pageind); + ret = atomic_read_p((void **)&elm->prof_tctx); + } } else ret = huge_prof_tctx_get(ptr); @@ -965,8 +968,11 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; assert(arena_mapbits_allocated_get(chunk, pageind) != 0); - if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) - arena_miscelm_get(chunk, pageind)->prof_tctx = tctx; + if (unlikely(arena_mapbits_large_get(chunk, pageind) != 0)) { + arena_chunk_map_misc_t *elm = arena_miscelm_get(chunk, + pageind); + atomic_write_p((void **)&elm->prof_tctx, tctx); + } } else huge_prof_tctx_set(ptr, tctx); } From ab5e3790f6bc2dc0c4d7c3d537387cf2563456ff Mon Sep 17 00:00:00 2001 From: Dan McGregor Date: Tue, 23 Dec 2014 16:09:32 -0600 Subject: [PATCH 225/352] Build docs in object directory --- Makefile.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile.in b/Makefile.in index da397c38..b1d88af9 100644 --- a/Makefile.in +++ b/Makefile.in @@ -104,8 +104,8 @@ endif PC := $(objroot)jemalloc.pc MAN3 := $(objroot)doc/jemalloc$(install_suffix).3 DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml -DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) -DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3) +DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.html) +DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(objroot)%.3) DOCS := $(DOCS_HTML) $(DOCS_MAN3) C_TESTLIB_SRCS := $(srcroot)test/src/btalloc.c $(srcroot)test/src/btalloc_0.c \ $(srcroot)test/src/btalloc_1.c $(srcroot)test/src/math.c \ @@ -181,10 +181,10 @@ all: build_lib dist: build_doc -$(srcroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl +$(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< -$(srcroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl +$(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< build_doc_html: $(DOCS_HTML) From f8880310ebb0ad5e1acce6e9886395e20041a32f Mon Sep 17 00:00:00 2001 From: Dan McGregor Date: Tue, 23 Dec 2014 16:10:08 -0600 Subject: [PATCH 226/352] Put VERSION file in object directory Also allow for the possibility that there exists a VERSION file in the srcroot, in case of building from a release tarball out of tree. --- Makefile.in | 2 +- configure.ac | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index b1d88af9..a105bb12 100644 --- a/Makefile.in +++ b/Makefile.in @@ -418,7 +418,7 @@ distclean: clean relclean: distclean rm -f $(objroot)configure - rm -f $(srcroot)VERSION + rm -f $(objroot)VERSION rm -f $(DOCS_HTML) rm -f $(DOCS_MAN3) diff --git a/configure.ac b/configure.ac index 2922880a..240d27af 100644 --- a/configure.ac +++ b/configure.ac @@ -1046,32 +1046,36 @@ dnl jemalloc configuration. dnl dnl Set VERSION if source directory is inside a git repository. -if test "x`git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then +if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then dnl Pattern globs aren't powerful enough to match both single- and dnl double-digit version numbers, so iterate over patterns to support up to dnl version 99.99.99 without any accidental matches. - rm -f "${srcroot}VERSION" + rm -f "${objroot}VERSION" for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ '[0-9][0-9].[0-9][0-9].[0-9]' \ '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do - if test ! -e "${srcroot}VERSION" ; then - git describe --long --abbrev=40 --match="${pattern}" > "${srcroot}VERSION.tmp" 2>/dev/null + if test ! -e "${objroot}VERSION" ; then + (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null if test $? -eq 0 ; then - mv "${srcroot}VERSION.tmp" "${srcroot}VERSION" + mv "${objroot}VERSION.tmp" "${objroot}VERSION" break fi fi done fi -rm -f "${srcroot}VERSION.tmp" -if test ! -e "${srcroot}VERSION" ; then - AC_MSG_RESULT( - [Missing VERSION file, and unable to generate it; creating bogus VERSION]) - echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${srcroot}VERSION" +rm -f "${objroot}VERSION.tmp" +if test ! -e "${objroot}VERSION" ; then + if test ! -e "${srcroot}VERSION" ; then + AC_MSG_RESULT( + [Missing VERSION file, and unable to generate it; creating bogus VERSION]) + echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" + else + cp ${srcroot}VERSION ${objroot}VERSION + fi fi -jemalloc_version=`cat "${srcroot}VERSION"` +jemalloc_version=`cat "${objroot}VERSION"` jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` From feaaa3df0da9972b9c5016c55b886e54853cc855 Mon Sep 17 00:00:00 2001 From: Abhishek Kulkarni Date: Wed, 11 Feb 2015 14:38:10 -0500 Subject: [PATCH 227/352] Take into account the install suffix that jemalloc was built with in the pkg-config file. Signed-off-by: Abhishek Kulkarni --- jemalloc.pc.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jemalloc.pc.in b/jemalloc.pc.in index af3f945d..1a3ad9b3 100644 --- a/jemalloc.pc.in +++ b/jemalloc.pc.in @@ -2,10 +2,11 @@ prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ +install_suffix=@install_suffix@ Name: jemalloc Description: A general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support. URL: http://www.canonware.com/jemalloc Version: @jemalloc_version@ Cflags: -I${includedir} -Libs: -L${libdir} -ljemalloc +Libs: -L${libdir} -ljemalloc${install_suffix} From 41cfe03f39740fe61cf46d86982f66c24168de32 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 13 Feb 2015 15:28:56 -0800 Subject: [PATCH 228/352] If MALLOCX_ARENA(a) is specified, use it during tcache fill. --- include/jemalloc/internal/arena.h | 26 ++++++++++++-------------- include/jemalloc/internal/tcache.h | 28 +++++++++++++++------------- src/tcache.c | 19 ++++++++++--------- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 4d88736d..b195daf0 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -985,28 +985,26 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, assert(size != 0); assert(size <= arena_maxclass); + arena = arena_choose(tsd, arena); + if (unlikely(arena == NULL)) + return (NULL); + if (likely(size <= SMALL_MAXCLASS)) { - if (likely(tcache != NULL)) - return (tcache_alloc_small(tsd, tcache, size, zero)); - else { - arena = arena_choose(tsd, arena); - if (unlikely(arena == NULL)) - return (NULL); + if (likely(tcache != NULL)) { + return (tcache_alloc_small(tsd, arena, tcache, size, + zero)); + } else return (arena_malloc_small(arena, size, zero)); - } } else if (likely(size <= arena_maxclass)) { /* * Initialize tcache after checking size in order to avoid * infinite recursion during tcache initialization. */ - if (likely(tcache != NULL) && size <= tcache_maxclass) - return (tcache_alloc_large(tsd, tcache, size, zero)); - else { - arena = arena_choose(tsd, arena); - if (unlikely(arena == NULL)) - return (NULL); + if (likely(tcache != NULL) && size <= tcache_maxclass) { + return (tcache_alloc_large(tsd, arena, tcache, size, + zero)); + } else return (arena_malloc_large(arena, size, zero)); - } } else return (huge_malloc(tsd, arena, size, zero, tcache)); } diff --git a/include/jemalloc/internal/tcache.h b/include/jemalloc/internal/tcache.h index 2a3952be..d2443b12 100644 --- a/include/jemalloc/internal/tcache.h +++ b/include/jemalloc/internal/tcache.h @@ -120,10 +120,10 @@ extern tcaches_t *tcaches; size_t tcache_salloc(const void *ptr); void tcache_event_hard(tsd_t *tsd, tcache_t *tcache); -void *tcache_alloc_small_hard(tsd_t *tsd, tcache_t *tcache, +void *tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache, tcache_bin_t *tbin, index_t binind); -void tcache_bin_flush_small(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, - unsigned rem, tcache_t *tcache); +void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, + index_t binind, unsigned rem); void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_associate(tcache_t *tcache, arena_t *arena); @@ -151,10 +151,10 @@ bool tcache_enabled_get(void); tcache_t *tcache_get(tsd_t *tsd, bool create); void tcache_enabled_set(bool enabled); void *tcache_alloc_easy(tcache_bin_t *tbin); -void *tcache_alloc_small(tsd_t *tsd, tcache_t *tcache, size_t size, - bool zero); -void *tcache_alloc_large(tsd_t *tsd, tcache_t *tcache, size_t size, - bool zero); +void *tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, + size_t size, bool zero); +void *tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, + size_t size, bool zero); void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, index_t binind); void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, @@ -258,7 +258,8 @@ tcache_alloc_easy(tcache_bin_t *tbin) } JEMALLOC_ALWAYS_INLINE void * -tcache_alloc_small(tsd_t *tsd, tcache_t *tcache, size_t size, bool zero) +tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, + bool zero) { void *ret; index_t binind; @@ -271,7 +272,7 @@ tcache_alloc_small(tsd_t *tsd, tcache_t *tcache, size_t size, bool zero) usize = index2size(binind); ret = tcache_alloc_easy(tbin); if (unlikely(ret == NULL)) { - ret = tcache_alloc_small_hard(tsd, tcache, tbin, binind); + ret = tcache_alloc_small_hard(tsd, arena, tcache, tbin, binind); if (ret == NULL) return (NULL); } @@ -302,7 +303,8 @@ tcache_alloc_small(tsd_t *tsd, tcache_t *tcache, size_t size, bool zero) } JEMALLOC_ALWAYS_INLINE void * -tcache_alloc_large(tsd_t *tsd, tcache_t *tcache, size_t size, bool zero) +tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, + bool zero) { void *ret; index_t binind; @@ -320,7 +322,7 @@ tcache_alloc_large(tsd_t *tsd, tcache_t *tcache, size_t size, bool zero) * Only allocate one large object at a time, because it's quite * expensive to create one and not use it. */ - ret = arena_malloc_large(arena_choose(tsd, NULL), usize, zero); + ret = arena_malloc_large(arena, usize, zero); if (ret == NULL) return (NULL); } else { @@ -366,8 +368,8 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, index_t binind) tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; if (unlikely(tbin->ncached == tbin_info->ncached_max)) { - tcache_bin_flush_small(tsd, tbin, binind, - (tbin_info->ncached_max >> 1), tcache); + tcache_bin_flush_small(tsd, tcache, tbin, binind, + (tbin_info->ncached_max >> 1)); } assert(tbin->ncached < tbin_info->ncached_max); tbin->avail[tbin->ncached] = ptr; diff --git a/src/tcache.c b/src/tcache.c index 10c85dd3..318e0dc8 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -41,8 +41,9 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) * Flush (ceiling) 3/4 of the objects below the low water mark. */ if (binind < NBINS) { - tcache_bin_flush_small(tsd, tbin, binind, tbin->ncached - - tbin->low_water + (tbin->low_water >> 2), tcache); + tcache_bin_flush_small(tsd, tcache, tbin, binind, + tbin->ncached - tbin->low_water + (tbin->low_water + >> 2)); } else { tcache_bin_flush_large(tsd, tbin, binind, tbin->ncached - tbin->low_water + (tbin->low_water >> 2), tcache); @@ -70,13 +71,13 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) } void * -tcache_alloc_small_hard(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, - index_t binind) +tcache_alloc_small_hard(tsd_t *tsd, arena_t *arena, tcache_t *tcache, + tcache_bin_t *tbin, index_t binind) { void *ret; - arena_tcache_fill_small(arena_choose(tsd, NULL), tbin, binind, - config_prof ? tcache->prof_accumbytes : 0); + arena_tcache_fill_small(arena, tbin, binind, config_prof ? + tcache->prof_accumbytes : 0); if (config_prof) tcache->prof_accumbytes = 0; ret = tcache_alloc_easy(tbin); @@ -85,8 +86,8 @@ tcache_alloc_small_hard(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, } void -tcache_bin_flush_small(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, - unsigned rem, tcache_t *tcache) +tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, + index_t binind, unsigned rem) { arena_t *arena; void *ptr; @@ -350,7 +351,7 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache) for (i = 0; i < NBINS; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_small(tsd, tbin, i, 0, tcache); + tcache_bin_flush_small(tsd, tcache, tbin, i, 0); if (config_stats && tbin->tstats.nrequests != 0) { arena_bin_t *bin = &arena->bins[i]; From b01186cebd9828e91a488d86980544bacb01e1a6 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 15 Feb 2015 14:04:55 -0800 Subject: [PATCH 229/352] Remove redundant tcache_boot() call. --- src/jemalloc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 3903209b..d5110092 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -1193,8 +1193,6 @@ malloc_init_hard_a0_locked(void) arena_boot(); if (config_tcache && tcache_boot()) return (true); - if (config_tcache && tcache_boot()) - malloc_mutex_unlock(&init_lock); if (malloc_mutex_init(&arenas_lock)) return (true); /* From 2195ba4e1f8f262b7e6586106d90f4dc0aea7630 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 15 Feb 2015 16:43:52 -0800 Subject: [PATCH 230/352] Normalize *_link and link_* fields to all be *_link. --- include/jemalloc/internal/extent.h | 6 +++--- src/arena.c | 8 ++++---- src/extent.c | 5 ++--- src/huge.c | 6 +++--- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index fbcdcf99..885f475b 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -34,14 +34,14 @@ struct extent_node_s { union { /* Linkage for the size/address-ordered tree. */ - rb_node(extent_node_t) link_szad; + rb_node(extent_node_t) szad_link; /* Linkage for huge allocations and cached chunks nodes. */ - ql_elm(extent_node_t) link_ql; + ql_elm(extent_node_t) ql_link; }; /* Linkage for the address-ordered tree. */ - rb_node(extent_node_t) link_ad; + rb_node(extent_node_t) ad_link; }; typedef rb_tree(extent_node_t) extent_tree_t; diff --git a/src/arena.c b/src/arena.c index 7b441bed..ce500f41 100644 --- a/src/arena.c +++ b/src/arena.c @@ -607,12 +607,12 @@ arena_node_alloc(arena_t *arena) extent_node_t *node; malloc_mutex_lock(&arena->node_cache_mtx); - node = ql_last(&arena->node_cache, link_ql); + node = ql_last(&arena->node_cache, ql_link); if (node == NULL) { malloc_mutex_unlock(&arena->node_cache_mtx); return (base_alloc(sizeof(extent_node_t))); } - ql_tail_remove(&arena->node_cache, extent_node_t, link_ql); + ql_tail_remove(&arena->node_cache, extent_node_t, ql_link); malloc_mutex_unlock(&arena->node_cache_mtx); return (node); } @@ -622,8 +622,8 @@ arena_node_dalloc(arena_t *arena, extent_node_t *node) { malloc_mutex_lock(&arena->node_cache_mtx); - ql_elm_new(node, link_ql); - ql_tail_insert(&arena->node_cache, node, link_ql); + ql_elm_new(node, ql_link); + ql_tail_insert(&arena->node_cache, node, ql_link); malloc_mutex_unlock(&arena->node_cache_mtx); } diff --git a/src/extent.c b/src/extent.c index ca852016..60e24683 100644 --- a/src/extent.c +++ b/src/extent.c @@ -22,7 +22,7 @@ extent_szad_comp(extent_node_t *a, extent_node_t *b) } /* Generate red-black tree functions. */ -rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad, +rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, szad_link, extent_szad_comp) JEMALLOC_INLINE_C int @@ -35,5 +35,4 @@ extent_ad_comp(extent_node_t *a, extent_node_t *b) } /* Generate red-black tree functions. */ -rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad, - extent_ad_comp) +rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, ad_link, extent_ad_comp) diff --git a/src/huge.c b/src/huge.c index 00327277..bc7d99cb 100644 --- a/src/huge.c +++ b/src/huge.c @@ -86,8 +86,8 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, /* Insert node into huge. */ malloc_mutex_lock(&arena->huge_mtx); - ql_elm_new(node, link_ql); - ql_tail_insert(&arena->huge, node, link_ql); + ql_elm_new(node, ql_link); + ql_tail_insert(&arena->huge, node, ql_link); malloc_mutex_unlock(&arena->huge_mtx); if (zero || (config_fill && unlikely(opt_zero))) { @@ -361,7 +361,7 @@ huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) arena = node->arena; huge_node_unset(ptr, node); malloc_mutex_lock(&arena->huge_mtx); - ql_remove(&arena->huge, node, link_ql); + ql_remove(&arena->huge, node, ql_link); malloc_mutex_unlock(&arena->huge_mtx); huge_dalloc_junk(node->addr, node->size); From 02e5dcf39d4995d2f37d0b18aa8511973938ac51 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 15 Feb 2015 20:12:06 -0800 Subject: [PATCH 231/352] Fix --enable-debug regression. Fix --enable-debug to actually enable debug mode. This regression was introduced by cbf3a6d70371d2390b8b0e76814e04cc6088002c (Move centralized chunk management into arenas.). --- configure.ac | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure.ac b/configure.ac index 240d27af..7a694a20 100644 --- a/configure.ac +++ b/configure.ac @@ -634,6 +634,9 @@ fi ], [enable_debug="0"] ) +if test "x$enable_debug" = "x1" ; then + AC_DEFINE([JEMALLOC_DEBUG], [ ]) +fi AC_SUBST([enable_debug]) dnl Only optimize if not debugging. From cb9b44914e7e25c6b08af7124d7f8f976e059555 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 15 Feb 2015 20:13:28 -0800 Subject: [PATCH 232/352] Remove obsolete (incorrect) assertions. This regression was introduced by 88fef7ceda6269598cef0cee8b984c8765673c27 (Refactor huge_*() calls into arena internals.), and went undetected because of the --enable-debug regression. --- include/jemalloc/internal/arena.h | 2 -- test/integration/mallocx.c | 45 ++++++++++++++++--------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index b195daf0..232e9a61 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -983,7 +983,6 @@ arena_malloc(tsd_t *tsd, arena_t *arena, size_t size, bool zero, { assert(size != 0); - assert(size <= arena_maxclass); arena = arena_choose(tsd, arena); if (unlikely(arena == NULL)) @@ -1031,7 +1030,6 @@ arena_salloc(const void *ptr, bool demote) index_t binind; assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) { diff --git a/test/integration/mallocx.c b/test/integration/mallocx.c index 123e041f..23129c20 100644 --- a/test/integration/mallocx.c +++ b/test/integration/mallocx.c @@ -2,34 +2,37 @@ #define CHUNK 0x400000 #define MAXALIGN (((size_t)1) << 25) +#define MAXSZ (((size_t)1) << 26) #define NITER 4 TEST_BEGIN(test_basic) { - size_t nsz, rsz, sz; - void *p; + size_t sz; - sz = 42; - nsz = nallocx(sz, 0); - assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); - p = mallocx(sz, 0); - assert_ptr_not_null(p, "Unexpected mallocx() error"); - rsz = sallocx(p, 0); - assert_zu_ge(rsz, sz, "Real size smaller than expected"); - assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch"); - dallocx(p, 0); + for (sz = 1; sz < MAXSZ; sz = nallocx(sz, 0) + 1) { + size_t nsz, rsz; + void *p; + nsz = nallocx(sz, 0); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + p = mallocx(sz, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + rsz = sallocx(p, 0); + assert_zu_ge(rsz, sz, "Real size smaller than expected"); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch"); + dallocx(p, 0); - p = mallocx(sz, 0); - assert_ptr_not_null(p, "Unexpected mallocx() error"); - dallocx(p, 0); + p = mallocx(sz, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + dallocx(p, 0); - nsz = nallocx(sz, MALLOCX_ZERO); - assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); - p = mallocx(sz, MALLOCX_ZERO); - assert_ptr_not_null(p, "Unexpected mallocx() error"); - rsz = sallocx(p, 0); - assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch"); - dallocx(p, 0); + nsz = nallocx(sz, MALLOCX_ZERO); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + p = mallocx(sz, MALLOCX_ZERO); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + rsz = sallocx(p, 0); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch"); + dallocx(p, 0); + } } TEST_END From 40ab8f98e42fda3816e2a993f136ec4770c202c7 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 15 Feb 2015 20:26:45 -0800 Subject: [PATCH 233/352] Remove more obsolete (incorrect) assertions. This regression was introduced by 88fef7ceda6269598cef0cee8b984c8765673c27 (Refactor huge_*() calls into arena internals.), and went undetected because of the --enable-debug regression. --- include/jemalloc/internal/arena.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 232e9a61..6341a867 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -934,7 +934,6 @@ arena_prof_tctx_get(const void *ptr) cassert(config_prof); assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) { @@ -961,7 +960,6 @@ arena_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) cassert(config_prof); assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) { From ee41ad409a43d12900a5a3108f6c14f84e4eb0eb Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sun, 15 Feb 2015 18:04:46 -0800 Subject: [PATCH 234/352] Integrate whole chunks into unused dirty page purging machinery. Extend per arena unused dirty page purging to manage unused dirty chunks in aaddtion to unused dirty runs. Rather than immediately unmapping deallocated chunks (or purging them in the --disable-munmap case), store them in a separate set of trees, chunks_[sz]ad_dirty. Preferrentially allocate dirty chunks. When excessive unused dirty pages accumulate, purge runs and chunks in ingegrated LRU order (and unmap chunks in the --enable-munmap case). Refactor extent_node_t to provide accessor functions. --- include/jemalloc/internal/arena.h | 64 ++- include/jemalloc/internal/chunk.h | 4 +- include/jemalloc/internal/extent.h | 135 +++++- .../jemalloc/internal/jemalloc_internal.h.in | 10 +- include/jemalloc/internal/private_symbols.txt | 15 + src/arena.c | 399 +++++++++++++----- src/base.c | 16 +- src/chunk.c | 146 ++++--- src/chunk_dss.c | 8 +- src/extent.c | 12 +- src/huge.c | 61 +-- src/tcache.c | 9 +- 12 files changed, 631 insertions(+), 248 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 6341a867..f967be3a 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -35,6 +35,7 @@ typedef struct arena_s arena_t; /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +#ifdef JEMALLOC_ARENA_STRUCTS_A struct arena_run_s { /* Index of bin this run is associated with. */ index_t binind; @@ -136,7 +137,7 @@ struct arena_chunk_map_misc_s { union { /* Linkage for list of dirty runs. */ - ql_elm(arena_chunk_map_misc_t) dr_link; + qr(arena_chunk_map_misc_t) rd_link; /* Profile counters, used for large object runs. */ prof_tctx_t *prof_tctx; @@ -147,14 +148,16 @@ struct arena_chunk_map_misc_s { }; typedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t; typedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; -typedef ql_head(arena_chunk_map_misc_t) arena_chunk_miscelms_t; +typedef qr(arena_chunk_map_misc_t) arena_chunk_miscelms_t; +#endif /* JEMALLOC_ARENA_STRUCTS_A */ +#ifdef JEMALLOC_ARENA_STRUCTS_B /* Arena chunk header. */ struct arena_chunk_s { /* - * The arena that owns the chunk is node.arena. This field as a whole - * is used by chunks_rtree to support both ivsalloc() and core-based - * debugging. + * A pointer to the arena that owns the chunk is stored within the node. + * This field as a whole is used by chunks_rtree to support both + * ivsalloc() and core-based debugging. */ extent_node_t node; @@ -309,13 +312,29 @@ struct arena_s { size_t ndirty; /* - * Size/address-ordered trees of this arena's available runs. The trees - * are used for first-best-fit run allocation. + * Size/address-ordered tree of this arena's available runs. The tree + * is used for first-best-fit run allocation. */ arena_avail_tree_t runs_avail; - /* List of dirty runs this arena manages. */ - arena_chunk_miscelms_t runs_dirty; + /* + * Unused dirty memory this arena manages. Dirty memory is conceptually + * tracked as an arbitrarily interleaved LRU of runs and chunks, but the + * list linkage is actually semi-duplicated in order to avoid extra + * arena_chunk_map_misc_t space overhead. + * + * LRU-----------------------------------------------------------MRU + * + * ______________ ___ ___ + * ...-->|chunks_dirty|<--------->|c|<-------------------->|c|<--... + * -------------- |h| |h| + * ____________ _____ |u| _____ _____ |u| + * ...-->|runs_dirty|<-->|run|<-->|n|<-->|run|<-->|run|<-->|n|<--... + * ------------ ----- |k| ----- ----- |k| + * --- --- + */ + arena_chunk_map_misc_t runs_dirty; + extent_node_t chunks_dirty; /* Extant huge allocations. */ ql_head(extent_node_t) huge; @@ -329,6 +348,8 @@ struct arena_s { * orderings are needed, which is why there are two trees with the same * contents. */ + extent_tree_t chunks_szad_dirty; + extent_tree_t chunks_ad_dirty; extent_tree_t chunks_szad_mmap; extent_tree_t chunks_ad_mmap; extent_tree_t chunks_szad_dss; @@ -347,6 +368,7 @@ struct arena_s { /* bins is used to store trees of free regions. */ arena_bin_t bins[NBINS]; }; +#endif /* JEMALLOC_ARENA_STRUCTS_B */ #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ @@ -363,6 +385,10 @@ extern size_t arena_maxclass; /* Max size class for arenas. */ extern unsigned nlclasses; /* Number of large size classes. */ extern unsigned nhclasses; /* Number of huge size classes. */ +void arena_chunk_dirty_maybe_insert(arena_t *arena, extent_node_t *node, + bool dirty); +void arena_chunk_dirty_maybe_remove(arena_t *arena, extent_node_t *node, + bool dirty); extent_node_t *arena_node_alloc(arena_t *arena); void arena_node_dalloc(arena_t *arena, extent_node_t *node); void *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, @@ -818,7 +844,7 @@ arena_ptr_small_binind_get(const void *ptr, size_t mapbits) assert(binind != BININD_INVALID); assert(binind < NBINS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->node.arena; + arena = extent_node_arena_get(&chunk->node); pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; actual_mapbits = arena_mapbits_get(chunk, pageind); assert(mapbits == actual_mapbits); @@ -1013,7 +1039,7 @@ arena_aalloc(const void *ptr) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (likely(chunk != ptr)) - return (chunk->node.arena); + return (extent_node_arena_get(&chunk->node)); else return (huge_aalloc(ptr)); } @@ -1085,8 +1111,8 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) mapbits); tcache_dalloc_small(tsd, tcache, ptr, binind); } else { - arena_dalloc_small(chunk->node.arena, chunk, - ptr, pageind); + arena_dalloc_small(extent_node_arena_get( + &chunk->node), chunk, ptr, pageind); } } else { size_t size = arena_mapbits_large_size_get(chunk, @@ -1097,8 +1123,8 @@ arena_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) if (likely(tcache != NULL) && size <= tcache_maxclass) tcache_dalloc_large(tsd, tcache, ptr, size); else { - arena_dalloc_large(chunk->node.arena, chunk, - ptr); + arena_dalloc_large(extent_node_arena_get( + &chunk->node), chunk, ptr); } } } else @@ -1136,8 +1162,8 @@ arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) } else { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - arena_dalloc_small(chunk->node.arena, chunk, - ptr, pageind); + arena_dalloc_small(extent_node_arena_get( + &chunk->node), chunk, ptr, pageind); } } else { assert(((uintptr_t)ptr & PAGE_MASK) == 0); @@ -1145,8 +1171,8 @@ arena_sdalloc(tsd_t *tsd, void *ptr, size_t size, tcache_t *tcache) if (likely(tcache != NULL) && size <= tcache_maxclass) tcache_dalloc_large(tsd, tcache, ptr, size); else { - arena_dalloc_large(chunk->node.arena, chunk, - ptr); + arena_dalloc_large(extent_node_arena_get( + &chunk->node), chunk, ptr); } } } else diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 5e0fb144..96b9e159 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -44,8 +44,10 @@ void *chunk_alloc_arena(chunk_alloc_t *chunk_alloc, size_t size, size_t alignment, bool *zero); void *chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind); -void chunk_unmap(arena_t *arena, void *chunk, size_t size); +void chunk_record(arena_t *arena, extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, bool dirty, void *chunk, size_t size); bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); +void chunk_unmap(arena_t *arena, void *chunk, size_t size); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 885f475b..10607614 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -7,36 +7,48 @@ typedef struct extent_node_s extent_node_t; /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS -/* Tree of extents. */ +/* Tree of extents. Use accessor functions for en_* fields. */ struct extent_node_s { /* Arena from which this extent came, if any. */ - arena_t *arena; + arena_t *en_arena; /* Pointer to the extent that this tree node is responsible for. */ - void *addr; + void *en_addr; + + /* Total region size. */ + size_t en_size; /* - * Total region size, or 0 if this node corresponds to an arena chunk. + * The zeroed flag is used by chunk recycling code to track whether + * memory is zero-filled. */ - size_t size; + bool en_zeroed; /* - * 'prof_tctx' and 'zeroed' are never needed at the same time, so - * overlay them in order to fit extent_node_t in one cache line. + * The achunk flag is used to validate that huge allocation lookups + * don't return arena chunks. */ + bool en_achunk; + union { /* Profile counters, used for huge objects. */ - prof_tctx_t *prof_tctx; + prof_tctx_t *en_prof_tctx; - /* True if zero-filled; used by chunk recycling code. */ - bool zeroed; + struct { + /* + * Linkage for arena's runs_dirty and chunks_dirty + * rings. + */ + qr(extent_node_t) cd_link; + arena_chunk_map_misc_t runs_dirty; + }; }; union { /* Linkage for the size/address-ordered tree. */ rb_node(extent_node_t) szad_link; - /* Linkage for huge allocations and cached chunks nodes. */ + /* Linkage for arena's huge and node_cache lists. */ ql_elm(extent_node_t) ql_link; }; @@ -57,6 +69,107 @@ rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) /******************************************************************************/ #ifdef JEMALLOC_H_INLINES +#ifndef JEMALLOC_ENABLE_INLINE +arena_t *extent_node_arena_get(const extent_node_t *node); +void *extent_node_addr_get(const extent_node_t *node); +size_t extent_node_size_get(const extent_node_t *node); +bool extent_node_zeroed_get(const extent_node_t *node); +bool extent_node_achunk_get(const extent_node_t *node); +prof_tctx_t *extent_node_prof_tctx_get(const extent_node_t *node); +void extent_node_arena_set(extent_node_t *node, arena_t *arena); +void extent_node_addr_set(extent_node_t *node, void *addr); +void extent_node_size_set(extent_node_t *node, size_t size); +void extent_node_zeroed_set(extent_node_t *node, bool zeroed); +void extent_node_achunk_set(extent_node_t *node, bool achunk); +void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) +JEMALLOC_INLINE arena_t * +extent_node_arena_get(const extent_node_t *node) +{ + + return (node->en_arena); +} + +JEMALLOC_INLINE void * +extent_node_addr_get(const extent_node_t *node) +{ + + return (node->en_addr); +} + +JEMALLOC_INLINE size_t +extent_node_size_get(const extent_node_t *node) +{ + + return (node->en_size); +} + +JEMALLOC_INLINE bool +extent_node_zeroed_get(const extent_node_t *node) +{ + + return (node->en_zeroed); +} + +JEMALLOC_INLINE bool +extent_node_achunk_get(const extent_node_t *node) +{ + + return (node->en_achunk); +} + +JEMALLOC_INLINE prof_tctx_t * +extent_node_prof_tctx_get(const extent_node_t *node) +{ + + return (node->en_prof_tctx); +} + +JEMALLOC_INLINE void +extent_node_arena_set(extent_node_t *node, arena_t *arena) +{ + + node->en_arena = arena; +} + +JEMALLOC_INLINE void +extent_node_addr_set(extent_node_t *node, void *addr) +{ + + node->en_addr = addr; +} + +JEMALLOC_INLINE void +extent_node_size_set(extent_node_t *node, size_t size) +{ + + node->en_size = size; +} + +JEMALLOC_INLINE void +extent_node_zeroed_set(extent_node_t *node, bool zeroed) +{ + + node->en_zeroed = zeroed; +} + +JEMALLOC_INLINE void +extent_node_achunk_set(extent_node_t *node, bool achunk) +{ + + node->en_achunk = achunk; +} + +JEMALLOC_INLINE void +extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx) +{ + + node->en_prof_tctx = tctx; +} +#endif + #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 43276c6c..8ed69ce2 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -368,8 +368,13 @@ typedef unsigned index_t; #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" -#include "jemalloc/internal/extent.h" +#define JEMALLOC_ARENA_STRUCTS_A #include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_STRUCTS_A +#include "jemalloc/internal/extent.h" +#define JEMALLOC_ARENA_STRUCTS_B +#include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_STRUCTS_B #include "jemalloc/internal/base.h" #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/chunk.h" @@ -933,7 +938,8 @@ ivsalloc(const void *ptr, bool demote) if (node == NULL) return (0); /* Only arena chunks should be looked up via interior pointers. */ - assert(node->addr == ptr || node->size == 0); + assert(extent_node_addr_get(node) == ptr || + extent_node_achunk_get(node)); return (isalloc(ptr, demote)); } diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index d5601a68..a1d12cf6 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -13,6 +13,8 @@ arena_choose arena_choose_hard arena_chunk_alloc_huge arena_chunk_dalloc_huge +arena_chunk_dirty_maybe_insert +arena_chunk_dirty_maybe_remove arena_chunk_ralloc_huge_expand arena_chunk_ralloc_huge_shrink arena_chunk_ralloc_huge_similar @@ -143,6 +145,7 @@ chunk_npages chunk_postfork_child chunk_postfork_parent chunk_prefork +chunk_record chunk_register chunk_unmap chunks_rtree @@ -173,6 +176,18 @@ ctl_postfork_child ctl_postfork_parent ctl_prefork dss_prec_names +extent_node_achunk_get +extent_node_achunk_set +extent_node_addr_get +extent_node_addr_set +extent_node_arena_get +extent_node_arena_set +extent_node_prof_tctx_get +extent_node_prof_tctx_set +extent_node_size_get +extent_node_size_set +extent_node_zeroed_get +extent_node_zeroed_set extent_tree_ad_empty extent_tree_ad_first extent_tree_ad_insert diff --git a/src/arena.c b/src/arena.c index ce500f41..a7a98e22 100644 --- a/src/arena.c +++ b/src/arena.c @@ -112,34 +112,94 @@ arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, } static void -arena_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, +arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, size_t npages) { arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - ql_elm_new(miscelm, dr_link); - ql_tail_insert(&arena->runs_dirty, miscelm, dr_link); + + qr_new(miscelm, rd_link); + qr_meld(&arena->runs_dirty, miscelm, rd_link); arena->ndirty += npages; } static void -arena_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, +arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, size_t npages) { arena_chunk_map_misc_t *miscelm = arena_miscelm_get(chunk, pageind); + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> LG_PAGE)); assert(arena_mapbits_dirty_get(chunk, pageind) == CHUNK_MAP_DIRTY); assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - ql_remove(&arena->runs_dirty, miscelm, dr_link); + + qr_remove(miscelm, rd_link); + assert(arena->ndirty >= npages); arena->ndirty -= npages; } +static size_t +arena_chunk_dirty_npages(const extent_node_t *node) +{ + + return (extent_node_size_get(node) >> LG_PAGE); +} + +static void +arena_chunk_dirty_node_init(extent_node_t *node) +{ + + qr_new(node, cd_link); + qr_new(&node->runs_dirty, rd_link); +} + +static void +arena_chunk_dirty_insert(arena_chunk_map_misc_t *runs_dirty, + extent_node_t *chunks_dirty, extent_node_t *node) +{ + + qr_meld(chunks_dirty, node, cd_link); + qr_meld(runs_dirty, &node->runs_dirty, rd_link); +} + +static void +arena_chunk_dirty_remove(extent_node_t *node) +{ + + qr_remove(node, cd_link); + qr_remove(&node->runs_dirty, rd_link); +} + +void +arena_chunk_dirty_maybe_insert(arena_t *arena, extent_node_t *node, bool dirty) +{ + + arena_chunk_dirty_node_init(node); + if (dirty) { + arena_chunk_dirty_insert(&arena->runs_dirty, + &arena->chunks_dirty, node); + arena->ndirty += arena_chunk_dirty_npages(node); + } +} + +void +arena_chunk_dirty_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) +{ + + if (dirty) { + arena_chunk_dirty_remove(node); + assert(arena->ndirty >= arena_chunk_dirty_npages(node)); + arena->ndirty -= arena_chunk_dirty_npages(node); + } +} + JEMALLOC_INLINE_C void * arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) { @@ -243,7 +303,7 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, arena_avail_remove(arena, chunk, run_ind, total_pages); if (flag_dirty != 0) - arena_dirty_remove(arena, chunk, run_ind, total_pages); + arena_run_dirty_remove(arena, chunk, run_ind, total_pages); arena_cactive_update(arena, need_pages, 0); arena->nactive += need_pages; @@ -256,7 +316,7 @@ arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, (rem_pages << LG_PAGE), flag_dirty); - arena_dirty_insert(arena, chunk, run_ind+need_pages, + arena_run_dirty_insert(arena, chunk, run_ind+need_pages, rem_pages); } else { arena_mapbits_unallocated_set(chunk, run_ind+need_pages, @@ -405,9 +465,10 @@ arena_chunk_alloc_internal(arena_t *arena, bool *zero) chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL, chunksize, chunksize, zero); if (chunk != NULL) { - chunk->node.arena = arena; - chunk->node.addr = chunk; - chunk->node.size = 0; /* Indicates this is an arena chunk. */ + extent_node_arena_set(&chunk->node, arena); + extent_node_addr_set(&chunk->node, chunk); + extent_node_size_set(&chunk->node, chunksize); + extent_node_achunk_set(&chunk->node, true); if (chunk_register(chunk, &chunk->node)) { chunk_dalloc((void *)chunk, chunksize, arena->ind); chunk = NULL; @@ -516,7 +577,7 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) arena->spare = chunk; if (arena_mapbits_dirty_get(spare, map_bias) != 0) { - arena_dirty_remove(arena, spare, map_bias, + arena_run_dirty_remove(arena, spare, map_bias, chunk_npages-map_bias); } chunk_dalloc = arena->chunk_dalloc; @@ -899,18 +960,29 @@ static size_t arena_dirty_count(arena_t *arena) { size_t ndirty = 0; - arena_chunk_map_misc_t *miscelm; - arena_chunk_t *chunk; - size_t pageind, npages; + arena_chunk_map_misc_t *runselm; + extent_node_t *chunkselm; - ql_foreach(miscelm, &arena->runs_dirty, dr_link) { - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); - pageind = arena_miscelm_to_pageind(miscelm); - assert(arena_mapbits_allocated_get(chunk, pageind) == 0); - assert(arena_mapbits_large_get(chunk, pageind) == 0); - assert(arena_mapbits_dirty_get(chunk, pageind) != 0); - npages = arena_mapbits_unallocated_size_get(chunk, pageind) >> - LG_PAGE; + for (runselm = qr_next(&arena->runs_dirty, rd_link), + chunkselm = qr_next(&arena->chunks_dirty, cd_link); + runselm != &arena->runs_dirty; runselm = qr_next(runselm, + rd_link)) { + size_t npages; + + if (runselm == &chunkselm->runs_dirty) { + npages = extent_node_size_get(chunkselm) >> LG_PAGE; + chunkselm = qr_next(chunkselm, cd_link); + } else { + arena_chunk_t *chunk = (arena_chunk_t + *)CHUNK_ADDR2BASE(runselm); + size_t pageind = arena_miscelm_to_pageind(runselm); + assert(arena_mapbits_allocated_get(chunk, pageind) == + 0); + assert(arena_mapbits_large_get(chunk, pageind) == 0); + assert(arena_mapbits_dirty_get(chunk, pageind) != 0); + npages = arena_mapbits_unallocated_size_get(chunk, + pageind) >> LG_PAGE; + } ndirty += npages; } @@ -939,41 +1011,94 @@ arena_compute_npurge(arena_t *arena, bool all) static size_t arena_stash_dirty(arena_t *arena, bool all, size_t npurge, - arena_chunk_miscelms_t *miscelms) + arena_chunk_map_misc_t *purge_runs_sentinel, + extent_node_t *purge_chunks_sentinel) { - arena_chunk_map_misc_t *miscelm; + arena_chunk_map_misc_t *runselm, *runselm_next; + extent_node_t *chunkselm; size_t nstashed = 0; - /* Add at least npurge pages to purge_list. */ - for (miscelm = ql_first(&arena->runs_dirty); miscelm != NULL; - miscelm = ql_first(&arena->runs_dirty)) { - arena_chunk_t *chunk = - (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); - size_t pageind = arena_miscelm_to_pageind(miscelm); - size_t run_size = arena_mapbits_unallocated_size_get(chunk, - pageind); - size_t npages = run_size >> LG_PAGE; - arena_run_t *run = &miscelm->run; + /* Stash at least npurge pages. */ + for (runselm = qr_next(&arena->runs_dirty, rd_link), + chunkselm = qr_next(&arena->chunks_dirty, cd_link); + runselm != &arena->runs_dirty; runselm = runselm_next) { + size_t npages; + runselm_next = qr_next(runselm, rd_link); - assert(pageind + npages <= chunk_npages); - assert(arena_mapbits_dirty_get(chunk, pageind) == - arena_mapbits_dirty_get(chunk, pageind+npages-1)); + if (runselm == &chunkselm->runs_dirty) { + extent_node_t *chunkselm_next, *tnode; + void *addr; + size_t size; + bool zeroed, zero; + UNUSED void *chunk; - /* - * If purging the spare chunk's run, make it available prior to - * allocation. - */ - if (chunk == arena->spare) - arena_chunk_alloc(arena); + chunkselm_next = qr_next(chunkselm, cd_link); + /* + * Cache contents of chunkselm prior to it being + * destroyed as a side effect of allocating the chunk. + */ + addr = extent_node_addr_get(chunkselm); + size = extent_node_size_get(chunkselm); + zeroed = extent_node_zeroed_get(chunkselm); + /* Allocate. */ + zero = false; + chunk = arena->chunk_alloc(addr, size, chunksize, &zero, + arena->ind); + assert(chunk == addr); + /* + * Create a temporary node to link into the ring of + * stashed allocations. + */ + tnode = arena_node_alloc(arena); + /* + * OOM shouldn't be possible because chunk allocation + * just cached a node. + */ + assert(tnode != NULL); + extent_node_arena_set(tnode, arena); + extent_node_addr_set(tnode, addr); + extent_node_size_set(tnode, size); + extent_node_zeroed_set(tnode, zeroed); + arena_chunk_dirty_node_init(tnode); + /* Stash. */ + arena_chunk_dirty_insert(purge_runs_sentinel, + purge_chunks_sentinel, tnode); + npages = size >> LG_PAGE; + chunkselm = chunkselm_next; + } else { + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); + size_t pageind = arena_miscelm_to_pageind(runselm); + arena_run_t *run = &runselm->run; + size_t run_size = + arena_mapbits_unallocated_size_get(chunk, pageind); - /* Temporarily allocate the free dirty run. */ - arena_run_split_large(arena, run, run_size, false); - /* Append to purge_list for later processing. */ - ql_elm_new(miscelm, dr_link); - ql_tail_insert(miscelms, miscelm, dr_link); + npages = run_size >> LG_PAGE; + + assert(pageind + npages <= chunk_npages); + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, pageind+npages-1)); + + /* + * If purging the spare chunk's run, make it available + * prior to allocation. + */ + if (chunk == arena->spare) + arena_chunk_alloc(arena); + + /* Temporarily allocate the free dirty run. */ + arena_run_split_large(arena, run, run_size, false); + /* Append to purge_runs for later processing. */ + if (false) + qr_new(runselm, rd_link); /* Redundant. */ + else { + assert(qr_next(runselm, rd_link) == runselm); + assert(qr_prev(runselm, rd_link) == runselm); + } + qr_meld(purge_runs_sentinel, runselm, rd_link); + } nstashed += npages; - if (!all && nstashed >= npurge) break; } @@ -982,52 +1107,66 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, } static size_t -arena_purge_stashed(arena_t *arena, arena_chunk_miscelms_t *miscelms) +arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, + extent_node_t *purge_chunks_sentinel) { size_t npurged, nmadvise; - arena_chunk_map_misc_t *miscelm; + arena_chunk_map_misc_t *runselm; + extent_node_t *chunkselm; if (config_stats) nmadvise = 0; npurged = 0; malloc_mutex_unlock(&arena->lock); + for (runselm = qr_next(purge_runs_sentinel, rd_link), + chunkselm = qr_next(purge_chunks_sentinel, cd_link); + runselm != purge_runs_sentinel; runselm = qr_next(runselm, + rd_link)) { + size_t npages; - ql_foreach(miscelm, miscelms, dr_link) { - arena_chunk_t *chunk; - size_t pageind, run_size, npages, flag_unzeroed, i; - bool unzeroed; + if (runselm == &chunkselm->runs_dirty) { + size_t size = extent_node_size_get(chunkselm); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(miscelm); - pageind = arena_miscelm_to_pageind(miscelm); - run_size = arena_mapbits_large_size_get(chunk, pageind); - npages = run_size >> LG_PAGE; + pages_purge(extent_node_addr_get(chunkselm), size); + npages = size >> LG_PAGE; + chunkselm = qr_next(chunkselm, cd_link); + } else { + arena_chunk_t *chunk; + size_t pageind, run_size, flag_unzeroed, i; + bool unzeroed; - assert(pageind + npages <= chunk_npages); - unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << - LG_PAGE)), run_size); - flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); + pageind = arena_miscelm_to_pageind(runselm); + run_size = arena_mapbits_large_size_get(chunk, pageind); + npages = run_size >> LG_PAGE; - /* - * Set the unzeroed flag for all pages, now that pages_purge() - * has returned whether the pages were zeroed as a side effect - * of purging. This chunk map modification is safe even though - * the arena mutex isn't currently owned by this thread, - * because the run is marked as allocated, thus protecting it - * from being modified by any other thread. As long as these - * writes don't perturb the first and last elements' - * CHUNK_MAP_ALLOCATED bits, behavior is well defined. - */ - for (i = 0; i < npages; i++) { - arena_mapbits_unzeroed_set(chunk, pageind+i, - flag_unzeroed); + assert(pageind + npages <= chunk_npages); + unzeroed = pages_purge((void *)((uintptr_t)chunk + + (pageind << LG_PAGE)), run_size); + flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; + + /* + * Set the unzeroed flag for all pages, now that + * pages_purge() has returned whether the pages were + * zeroed as a side effect of purging. This chunk map + * modification is safe even though the arena mutex + * isn't currently owned by this thread, because the run + * is marked as allocated, thus protecting it from being + * modified by any other thread. As long as these + * writes don't perturb the first and last elements' + * CHUNK_MAP_ALLOCATED bits, behavior is well defined. + */ + for (i = 0; i < npages; i++) { + arena_mapbits_unzeroed_set(chunk, pageind+i, + flag_unzeroed); + } } npurged += npages; if (config_stats) nmadvise++; } - malloc_mutex_lock(&arena->lock); if (config_stats) { @@ -1039,16 +1178,31 @@ arena_purge_stashed(arena_t *arena, arena_chunk_miscelms_t *miscelms) } static void -arena_unstash_purged(arena_t *arena, arena_chunk_miscelms_t *miscelms) +arena_unstash_purged(arena_t *arena, + arena_chunk_map_misc_t *purge_runs_sentinel, + extent_node_t *purge_chunks_sentinel) { - arena_chunk_map_misc_t *miscelm; + arena_chunk_map_misc_t *runselm, *runselm_next; + extent_node_t *chunkselm; /* Deallocate runs. */ - for (miscelm = ql_first(miscelms); miscelm != NULL; - miscelm = ql_first(miscelms)) { - arena_run_t *run = &miscelm->run; - ql_remove(miscelms, miscelm, dr_link); - arena_run_dalloc(arena, run, false, true); + for (runselm = qr_next(purge_runs_sentinel, rd_link), + chunkselm = qr_next(purge_chunks_sentinel, cd_link); + runselm != purge_runs_sentinel; runselm = runselm_next) { + runselm_next = qr_next(runselm, rd_link); + if (runselm == &chunkselm->runs_dirty) { + extent_node_t *chunkselm_next = qr_next(chunkselm, + cd_link); + arena_chunk_dirty_remove(chunkselm); + chunk_unmap(arena, extent_node_addr_get(chunkselm), + extent_node_size_get(chunkselm)); + arena_node_dalloc(arena, chunkselm); + chunkselm = chunkselm_next; + } else { + arena_run_t *run = &runselm->run; + qr_remove(runselm, rd_link); + arena_run_dalloc(arena, run, false, true); + } } } @@ -1056,7 +1210,8 @@ void arena_purge(arena_t *arena, bool all) { size_t npurge, npurgeable, npurged; - arena_chunk_miscelms_t purge_list; + arena_chunk_map_misc_t purge_runs_sentinel; + extent_node_t purge_chunks_sentinel; /* * Calls to arena_dirty_count() are disabled even for debug builds @@ -1072,12 +1227,17 @@ arena_purge(arena_t *arena, bool all) arena->stats.npurge++; npurge = arena_compute_npurge(arena, all); - ql_new(&purge_list); - npurgeable = arena_stash_dirty(arena, all, npurge, &purge_list); + qr_new(&purge_runs_sentinel, rd_link); + arena_chunk_dirty_node_init(&purge_chunks_sentinel); + + npurgeable = arena_stash_dirty(arena, all, npurge, &purge_runs_sentinel, + &purge_chunks_sentinel); assert(npurgeable >= npurge); - npurged = arena_purge_stashed(arena, &purge_list); + npurged = arena_purge_stashed(arena, &purge_runs_sentinel, + &purge_chunks_sentinel); assert(npurged == npurgeable); - arena_unstash_purged(arena, &purge_list); + arena_unstash_purged(arena, &purge_runs_sentinel, + &purge_chunks_sentinel); } void @@ -1115,9 +1275,12 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, run_ind+run_pages+nrun_pages-1) == flag_dirty); arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages); - /* If the successor is dirty, remove it from runs_dirty. */ + /* + * If the successor is dirty, remove it from the set of dirty + * pages. + */ if (flag_dirty != 0) { - arena_dirty_remove(arena, chunk, run_ind+run_pages, + arena_run_dirty_remove(arena, chunk, run_ind+run_pages, nrun_pages); } @@ -1148,9 +1311,14 @@ arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); arena_avail_remove(arena, chunk, run_ind, prun_pages); - /* If the predecessor is dirty, remove it from runs_dirty. */ - if (flag_dirty != 0) - arena_dirty_remove(arena, chunk, run_ind, prun_pages); + /* + * If the predecessor is dirty, remove it from the set of dirty + * pages. + */ + if (flag_dirty != 0) { + arena_run_dirty_remove(arena, chunk, run_ind, + prun_pages); + } size += prun_size; run_pages += prun_pages; @@ -1224,7 +1392,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) arena_avail_insert(arena, chunk, run_ind, run_pages); if (dirty) - arena_dirty_insert(arena, chunk, run_ind, run_pages); + arena_run_dirty_insert(arena, chunk, run_ind, run_pages); /* Deallocate chunk if it is now completely unused. */ if (size == arena_maxrun) { @@ -1843,7 +2011,8 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, if (run == bin->runcur) bin->runcur = NULL; else { - index_t binind = arena_bin_index(chunk->node.arena, bin); + index_t binind = arena_bin_index(extent_node_arena_get( + &chunk->node), bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; if (bin_info->nregs != 1) { @@ -2184,7 +2353,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, arena_t *arena; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->node.arena; + arena = extent_node_arena_get(&chunk->node); if (usize < oldsize) { /* Fill before shrinking in order avoid a race. */ @@ -2422,20 +2591,6 @@ arena_new(unsigned ind) arena->nthreads = 0; if (malloc_mutex_init(&arena->lock)) return (NULL); - arena->chunk_alloc = chunk_alloc_default; - arena->chunk_dalloc = chunk_dalloc_default; - ql_new(&arena->huge); - if (malloc_mutex_init(&arena->huge_mtx)) - return (NULL); - extent_tree_szad_new(&arena->chunks_szad_mmap); - extent_tree_ad_new(&arena->chunks_ad_mmap); - extent_tree_szad_new(&arena->chunks_szad_dss); - extent_tree_ad_new(&arena->chunks_ad_dss); - ql_new(&arena->node_cache); - if (malloc_mutex_init(&arena->chunks_mtx)) - return (NULL); - if (malloc_mutex_init(&arena->node_cache_mtx)) - return (NULL); if (config_stats) { memset(&arena->stats, 0, sizeof(arena_stats_t)); @@ -2463,7 +2618,27 @@ arena_new(unsigned ind) arena->ndirty = 0; arena_avail_tree_new(&arena->runs_avail); - ql_new(&arena->runs_dirty); + qr_new(&arena->runs_dirty, rd_link); + qr_new(&arena->chunks_dirty, cd_link); + + ql_new(&arena->huge); + if (malloc_mutex_init(&arena->huge_mtx)) + return (NULL); + + extent_tree_szad_new(&arena->chunks_szad_dirty); + extent_tree_ad_new(&arena->chunks_ad_dirty); + extent_tree_szad_new(&arena->chunks_szad_mmap); + extent_tree_ad_new(&arena->chunks_ad_mmap); + extent_tree_szad_new(&arena->chunks_szad_dss); + extent_tree_ad_new(&arena->chunks_ad_dss); + if (malloc_mutex_init(&arena->chunks_mtx)) + return (NULL); + ql_new(&arena->node_cache); + if (malloc_mutex_init(&arena->node_cache_mtx)) + return (NULL); + + arena->chunk_alloc = chunk_alloc_default; + arena->chunk_dalloc = chunk_dalloc_default; /* Initialize bins. */ for (i = 0; i < NBINS; i++) { diff --git a/src/base.c b/src/base.c index 7b5804ee..819fa025 100644 --- a/src/base.c +++ b/src/base.c @@ -60,8 +60,8 @@ base_chunk_alloc(size_t minsize) if (config_stats) base_allocated += nsize; } - node->addr = addr; - node->size = csize; + extent_node_addr_set(node, addr); + extent_node_size_set(node, csize); return (node); } @@ -84,8 +84,8 @@ base_alloc(size_t size) */ csize = CACHELINE_CEILING(size); - key.addr = NULL; - key.size = csize; + extent_node_addr_set(&key, NULL); + extent_node_size_set(&key, csize); malloc_mutex_lock(&base_mtx); node = extent_tree_szad_nsearch(&base_avail_szad, &key); if (node != NULL) { @@ -100,10 +100,10 @@ base_alloc(size_t size) goto label_return; } - ret = node->addr; - if (node->size > csize) { - node->addr = (void *)((uintptr_t)ret + csize); - node->size -= csize; + ret = extent_node_addr_get(node); + if (extent_node_size_get(node) > csize) { + extent_node_addr_set(node, (void *)((uintptr_t)ret + csize)); + extent_node_size_set(node, extent_node_size_get(node) - csize); extent_tree_szad_insert(&base_avail_szad, node); } else base_node_dalloc(node); diff --git a/src/chunk.c b/src/chunk.c index b3576195..8bc87beb 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -24,12 +24,13 @@ bool chunk_register(const void *chunk, const extent_node_t *node) { - assert(node->addr == chunk); + assert(extent_node_addr_get(node) == chunk); if (rtree_set(&chunks_rtree, (uintptr_t)chunk, node)) return (true); if (config_prof && opt_prof) { - size_t nadd = (node->size == 0) ? 1 : node->size / chunksize; + size_t size = extent_node_size_get(node); + size_t nadd = (size == 0) ? 1 : size / chunksize; size_t cur = atomic_add_z(&curchunks, nadd); size_t high = atomic_read_z(&highchunks); while (cur > high && atomic_cas_z(&highchunks, high, cur)) { @@ -54,7 +55,8 @@ chunk_deregister(const void *chunk, const extent_node_t *node) err = rtree_set(&chunks_rtree, (uintptr_t)chunk, NULL); assert(!err); if (config_prof && opt_prof) { - size_t nsub = (node->size == 0) ? 1 : node->size / chunksize; + size_t size = extent_node_size_get(node); + size_t nsub = (size == 0) ? 1 : size / chunksize; assert(atomic_read_z(&curchunks) >= nsub); atomic_sub_z(&curchunks, nsub); } @@ -62,8 +64,8 @@ chunk_deregister(const void *chunk, const extent_node_t *node) static void * chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, void *new_addr, size_t size, size_t alignment, - bool *zero) + extent_tree_t *chunks_ad, bool dirty, void *new_addr, size_t size, + size_t alignment, bool *zero) { void *ret; extent_node_t *node; @@ -77,32 +79,35 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); - key.addr = new_addr; - key.size = alloc_size; + extent_node_addr_set(&key, new_addr); + extent_node_size_set(&key, alloc_size); malloc_mutex_lock(&arena->chunks_mtx); node = (new_addr != NULL) ? extent_tree_ad_search(chunks_ad, &key) : extent_tree_szad_nsearch(chunks_szad, &key); - if (node == NULL || (new_addr != NULL && node->size < size)) { + if (node == NULL || (new_addr != NULL && extent_node_size_get(node) < + size)) { malloc_mutex_unlock(&arena->chunks_mtx); return (NULL); } - leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) - - (uintptr_t)node->addr; + leadsize = ALIGNMENT_CEILING((uintptr_t)extent_node_addr_get(node), + alignment) - (uintptr_t)extent_node_addr_get(node); assert(new_addr == NULL || leadsize == 0); - assert(node->size >= leadsize + size); - trailsize = node->size - leadsize - size; - ret = (void *)((uintptr_t)node->addr + leadsize); - zeroed = node->zeroed; + assert(extent_node_size_get(node) >= leadsize + size); + trailsize = extent_node_size_get(node) - leadsize - size; + ret = (void *)((uintptr_t)extent_node_addr_get(node) + leadsize); + zeroed = extent_node_zeroed_get(node); if (zeroed) *zero = true; /* Remove node from the tree. */ extent_tree_szad_remove(chunks_szad, node); extent_tree_ad_remove(chunks_ad, node); + arena_chunk_dirty_maybe_remove(arena, node, dirty); if (leadsize != 0) { /* Insert the leading space as a smaller chunk. */ - node->size = leadsize; + extent_node_size_set(node, leadsize); extent_tree_szad_insert(chunks_szad, node); extent_tree_ad_insert(chunks_ad, node); + arena_chunk_dirty_maybe_insert(arena, node, dirty); node = NULL; } if (trailsize != 0) { @@ -111,15 +116,17 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, node = arena_node_alloc(arena); if (node == NULL) { malloc_mutex_unlock(&arena->chunks_mtx); - chunk_unmap(arena, ret, size); + chunk_record(arena, chunks_szad, chunks_ad, + dirty, ret, size); return (NULL); } } - node->addr = (void *)((uintptr_t)(ret) + size); - node->size = trailsize; - node->zeroed = zeroed; + extent_node_addr_set(node, (void *)((uintptr_t)(ret) + size)); + extent_node_size_set(node, trailsize); + extent_node_zeroed_set(node, zeroed); extent_tree_szad_insert(chunks_szad, node); extent_tree_ad_insert(chunks_ad, node); + arena_chunk_dirty_maybe_insert(arena, node, dirty); node = NULL; } malloc_mutex_unlock(&arena->chunks_mtx); @@ -148,7 +155,8 @@ chunk_alloc_core_dss(arena_t *arena, void *new_addr, size_t size, void *ret; if ((ret = chunk_recycle(arena, &arena->chunks_szad_dss, - &arena->chunks_ad_dss, new_addr, size, alignment, zero)) != NULL) + &arena->chunks_ad_dss, false, new_addr, size, alignment, zero)) != + NULL) return (ret); ret = chunk_alloc_dss(arena, new_addr, size, alignment, zero); return (ret); @@ -171,6 +179,11 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment, assert(alignment != 0); assert((alignment & chunksize_mask) == 0); + /* dirty. */ + if ((ret = chunk_recycle(arena, &arena->chunks_szad_dirty, + &arena->chunks_ad_dirty, true, new_addr, size, alignment, zero)) != + NULL) + return (ret); /* "primary" dss. */ if (have_dss && dss_prec == dss_prec_primary && (ret = chunk_alloc_core_dss(arena, new_addr, size, alignment, zero)) != @@ -178,8 +191,8 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment, return (ret); /* mmap. */ if (!config_munmap && (ret = chunk_recycle(arena, - &arena->chunks_szad_mmap, &arena->chunks_ad_mmap, new_addr, size, - alignment, zero)) != NULL) + &arena->chunks_szad_mmap, &arena->chunks_ad_mmap, false, new_addr, + size, alignment, zero)) != NULL) return (ret); /* * Requesting an address is not implemented for chunk_alloc_mmap(), so @@ -263,54 +276,62 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, arena->dss_prec)); } -static void +void chunk_record(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, void *chunk, size_t size) + extent_tree_t *chunks_ad, bool dirty, void *chunk, size_t size) { bool unzeroed; - extent_node_t *node, *prev, key; + extent_node_t *node, *prev; + extent_node_t key; - unzeroed = pages_purge(chunk, size); + unzeroed = dirty ? true : pages_purge(chunk, size); JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); malloc_mutex_lock(&arena->chunks_mtx); - key.addr = (void *)((uintptr_t)chunk + size); + extent_node_addr_set(&key, (void *)((uintptr_t)chunk + size)); node = extent_tree_ad_nsearch(chunks_ad, &key); /* Try to coalesce forward. */ - if (node != NULL && node->addr == key.addr) { + if (node != NULL && extent_node_addr_get(node) == + extent_node_addr_get(&key)) { /* * Coalesce chunk with the following address range. This does * not change the position within chunks_ad, so only * remove/insert from/into chunks_szad. */ extent_tree_szad_remove(chunks_szad, node); - node->addr = chunk; - node->size += size; - node->zeroed = (node->zeroed && !unzeroed); + arena_chunk_dirty_maybe_remove(arena, node, dirty); + extent_node_addr_set(node, chunk); + extent_node_size_set(node, extent_node_size_get(node) + size); + extent_node_zeroed_set(node, extent_node_zeroed_get(node) && + !unzeroed); extent_tree_szad_insert(chunks_szad, node); + arena_chunk_dirty_maybe_insert(arena, node, dirty); } else { /* Coalescing forward failed, so insert a new node. */ node = arena_node_alloc(arena); if (node == NULL) { /* * Node allocation failed, which is an exceedingly - * unlikely failure. Leak chunk; its pages have - * already been purged, so this is only a virtual - * memory leak. + * unlikely failure. Leak chunk after making sure its + * pages have already been purged, so that this is only + * a virtual memory leak. */ + if (dirty) + pages_purge(chunk, size); goto label_return; } - node->addr = chunk; - node->size = size; - node->zeroed = !unzeroed; + extent_node_addr_set(node, chunk); + extent_node_size_set(node, size); + extent_node_zeroed_set(node, !unzeroed); extent_tree_ad_insert(chunks_ad, node); extent_tree_szad_insert(chunks_szad, node); + arena_chunk_dirty_maybe_insert(arena, node, dirty); } /* Try to coalesce backward. */ prev = extent_tree_ad_prev(chunks_ad, node); - if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == - chunk) { + if (prev != NULL && (void *)((uintptr_t)extent_node_addr_get(prev) + + extent_node_size_get(prev)) == chunk) { /* * Coalesce chunk with the previous address range. This does * not change the position within chunks_ad, so only @@ -318,12 +339,16 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, */ extent_tree_szad_remove(chunks_szad, prev); extent_tree_ad_remove(chunks_ad, prev); - + arena_chunk_dirty_maybe_remove(arena, prev, dirty); extent_tree_szad_remove(chunks_szad, node); - node->addr = prev->addr; - node->size += prev->size; - node->zeroed = (node->zeroed && prev->zeroed); + arena_chunk_dirty_maybe_remove(arena, node, dirty); + extent_node_addr_set(node, extent_node_addr_get(prev)); + extent_node_size_set(node, extent_node_size_get(node) + + extent_node_size_get(prev)); + extent_node_zeroed_set(node, extent_node_zeroed_get(node) && + extent_node_zeroed_get(prev)); extent_tree_szad_insert(chunks_szad, node); + arena_chunk_dirty_maybe_insert(arena, node, dirty); arena_node_dalloc(arena, prev); } @@ -332,6 +357,28 @@ label_return: malloc_mutex_unlock(&arena->chunks_mtx); } +static void +chunk_cache(arena_t *arena, void *chunk, size_t size) +{ + + assert(chunk != NULL); + assert(CHUNK_ADDR2BASE(chunk) == chunk); + assert(size != 0); + assert((size & chunksize_mask) == 0); + + chunk_record(arena, &arena->chunks_szad_dirty, &arena->chunks_ad_dirty, + true, chunk, size); +} + +/* Default arena chunk deallocation routine in the absence of user override. */ +bool +chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) +{ + + chunk_cache(chunk_arena_get(arena_ind), chunk, size); + return (false); +} + void chunk_unmap(arena_t *arena, void *chunk, size_t size) { @@ -343,22 +390,13 @@ chunk_unmap(arena_t *arena, void *chunk, size_t size) if (have_dss && chunk_in_dss(chunk)) { chunk_record(arena, &arena->chunks_szad_dss, - &arena->chunks_ad_dss, chunk, size); + &arena->chunks_ad_dss, false, chunk, size); } else if (chunk_dalloc_mmap(chunk, size)) { chunk_record(arena, &arena->chunks_szad_mmap, - &arena->chunks_ad_mmap, chunk, size); + &arena->chunks_ad_mmap, false, chunk, size); } } -/* Default arena chunk deallocation routine in the absence of user override. */ -bool -chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) -{ - - chunk_unmap(chunk_arena_get(arena_ind), chunk, size); - return (false); -} - static rtree_node_elm_t * chunks_rtree_node_alloc(size_t nelms) { diff --git a/src/chunk_dss.c b/src/chunk_dss.c index 9c3eea82..c3c48481 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -133,8 +133,12 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, /* Success. */ dss_max = dss_next; malloc_mutex_unlock(&dss_mtx); - if (cpad_size != 0) - chunk_unmap(arena, cpad, cpad_size); + if (cpad_size != 0) { + chunk_record(arena, + &arena->chunks_szad_dss, + &arena->chunks_ad_dss, false, cpad, + cpad_size); + } if (*zero) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( ret, size); diff --git a/src/extent.c b/src/extent.c index 60e24683..f98e77e2 100644 --- a/src/extent.c +++ b/src/extent.c @@ -7,13 +7,13 @@ JEMALLOC_INLINE_C int extent_szad_comp(extent_node_t *a, extent_node_t *b) { int ret; - size_t a_size = a->size; - size_t b_size = b->size; + size_t a_size = extent_node_size_get(a); + size_t b_size = extent_node_size_get(b); ret = (a_size > b_size) - (a_size < b_size); if (ret == 0) { - uintptr_t a_addr = (uintptr_t)a->addr; - uintptr_t b_addr = (uintptr_t)b->addr; + uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a); + uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b); ret = (a_addr > b_addr) - (a_addr < b_addr); } @@ -28,8 +28,8 @@ rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, szad_link, JEMALLOC_INLINE_C int extent_ad_comp(extent_node_t *a, extent_node_t *b) { - uintptr_t a_addr = (uintptr_t)a->addr; - uintptr_t b_addr = (uintptr_t)b->addr; + uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a); + uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b); return ((a_addr > b_addr) - (a_addr < b_addr)); } diff --git a/src/huge.c b/src/huge.c index bc7d99cb..b9cae001 100644 --- a/src/huge.c +++ b/src/huge.c @@ -9,7 +9,7 @@ huge_node_get(const void *ptr) extent_node_t *node; node = chunk_lookup(ptr); - assert(node->size != 0); + assert(!extent_node_achunk_get(node)); return (node); } @@ -18,8 +18,8 @@ static bool huge_node_set(const void *ptr, extent_node_t *node) { - assert(node->addr == ptr); - assert(node->size != 0); + assert(extent_node_addr_get(node) == ptr); + assert(!extent_node_achunk_get(node)); return (chunk_register(ptr, node)); } @@ -73,10 +73,11 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, return (NULL); } - node->addr = ret; - node->size = usize; - node->zeroed = is_zeroed; - node->arena = arena; + extent_node_arena_set(node, arena); + extent_node_addr_set(node, ret); + extent_node_size_set(node, usize); + extent_node_achunk_set(node, false); + extent_node_zeroed_set(node, is_zeroed); if (huge_node_set(ret, node)) { arena_chunk_dalloc_huge(arena, ret, usize); @@ -152,13 +153,13 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, zeroed = true; node = huge_node_get(ptr); - arena = node->arena; + arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ - assert(node->size != usize); - node->size = usize; - /* Clear node->zeroed if zeroing failed above. */ - node->zeroed = (node->zeroed && zeroed); + assert(extent_node_size_get(node) != usize); + extent_node_size_set(node, usize); + /* Clear node's zeroed field if zeroing failed above. */ + extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed); malloc_mutex_unlock(&arena->huge_mtx); arena_chunk_ralloc_huge_similar(arena, ptr, oldsize, usize); @@ -195,12 +196,12 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) } node = huge_node_get(ptr); - arena = node->arena; + arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ - node->size = usize; - /* Clear node->zeroed if zeroing failed above. */ - node->zeroed = (node->zeroed && zeroed); + extent_node_size_set(node, usize); + /* Clear node's zeroed field if zeroing failed above. */ + extent_node_zeroed_set(node, extent_node_zeroed_get(node) && zeroed); malloc_mutex_unlock(&arena->huge_mtx); /* Zap the excess chunks. */ @@ -221,9 +222,9 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { } node = huge_node_get(ptr); - arena = node->arena; + arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); - is_zeroed_subchunk = node->zeroed; + is_zeroed_subchunk = extent_node_zeroed_get(node); malloc_mutex_unlock(&arena->huge_mtx); /* @@ -238,7 +239,7 @@ huge_ralloc_no_move_expand(void *ptr, size_t oldsize, size_t size, bool zero) { malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ - node->size = usize; + extent_node_size_set(node, usize); malloc_mutex_unlock(&arena->huge_mtx); if (zero || (config_fill && unlikely(opt_zero))) { @@ -358,14 +359,16 @@ huge_dalloc(tsd_t *tsd, void *ptr, tcache_t *tcache) arena_t *arena; node = huge_node_get(ptr); - arena = node->arena; + arena = extent_node_arena_get(node); huge_node_unset(ptr, node); malloc_mutex_lock(&arena->huge_mtx); ql_remove(&arena->huge, node, ql_link); malloc_mutex_unlock(&arena->huge_mtx); - huge_dalloc_junk(node->addr, node->size); - arena_chunk_dalloc_huge(node->arena, node->addr, node->size); + huge_dalloc_junk(extent_node_addr_get(node), + extent_node_size_get(node)); + arena_chunk_dalloc_huge(extent_node_arena_get(node), + extent_node_addr_get(node), extent_node_size_get(node)); idalloctm(tsd, node, tcache, true); } @@ -373,7 +376,7 @@ arena_t * huge_aalloc(const void *ptr) { - return (huge_node_get(ptr)->arena); + return (extent_node_arena_get(huge_node_get(ptr))); } size_t @@ -384,9 +387,9 @@ huge_salloc(const void *ptr) arena_t *arena; node = huge_node_get(ptr); - arena = node->arena; + arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); - size = node->size; + size = extent_node_size_get(node); malloc_mutex_unlock(&arena->huge_mtx); return (size); @@ -400,9 +403,9 @@ huge_prof_tctx_get(const void *ptr) arena_t *arena; node = huge_node_get(ptr); - arena = node->arena; + arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); - tctx = node->prof_tctx; + tctx = extent_node_prof_tctx_get(node); malloc_mutex_unlock(&arena->huge_mtx); return (tctx); @@ -415,8 +418,8 @@ huge_prof_tctx_set(const void *ptr, prof_tctx_t *tctx) arena_t *arena; node = huge_node_get(ptr); - arena = node->arena; + arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); - node->prof_tctx = tctx; + extent_node_prof_tctx_set(node, tctx); malloc_mutex_unlock(&arena->huge_mtx); } diff --git a/src/tcache.c b/src/tcache.c index 318e0dc8..8d0a6fa8 100644 --- a/src/tcache.c +++ b/src/tcache.c @@ -103,7 +103,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, /* Lock the arena bin associated with the first object. */ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); - arena_t *bin_arena = chunk->node.arena; + arena_t *bin_arena = extent_node_arena_get(&chunk->node); arena_bin_t *bin = &bin_arena->bins[binind]; if (config_prof && bin_arena == arena) { @@ -125,7 +125,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->node.arena == bin_arena) { + if (extent_node_arena_get(&chunk->node) == bin_arena) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_bits_t *bitselm = @@ -183,7 +183,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, /* Lock the arena associated with the first object. */ arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); - arena_t *locked_arena = chunk->node.arena; + arena_t *locked_arena = extent_node_arena_get(&chunk->node); UNUSED bool idump; if (config_prof) @@ -209,7 +209,8 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, index_t binind, ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->node.arena == locked_arena) { + if (extent_node_arena_get(&chunk->node) == + locked_arena) { arena_dalloc_large_junked_locked(locked_arena, chunk, ptr); } else { From a4e1888d1a12d864f42350f2859e33eb3a0033f2 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 17 Feb 2015 15:13:52 -0800 Subject: [PATCH 235/352] Simplify extent_node_t and add extent_node_init(). --- include/jemalloc/internal/extent.h | 33 ++++++++++++------- include/jemalloc/internal/private_symbols.txt | 1 + src/arena.c | 9 ++--- src/base.c | 6 ++-- src/chunk.c | 25 +++++++------- src/huge.c | 6 +--- 6 files changed, 38 insertions(+), 42 deletions(-) diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 10607614..9ee1b445 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -30,19 +30,12 @@ struct extent_node_s { */ bool en_achunk; - union { - /* Profile counters, used for huge objects. */ - prof_tctx_t *en_prof_tctx; + /* Profile counters, used for huge objects. */ + prof_tctx_t *en_prof_tctx; - struct { - /* - * Linkage for arena's runs_dirty and chunks_dirty - * rings. - */ - qr(extent_node_t) cd_link; - arena_chunk_map_misc_t runs_dirty; - }; - }; + /* Linkage for arena's runs_dirty and chunks_dirty rings. */ + qr(extent_node_t) cd_link; + arena_chunk_map_misc_t runs_dirty; union { /* Linkage for the size/address-ordered tree. */ @@ -82,6 +75,8 @@ void extent_node_size_set(extent_node_t *node, size_t size); void extent_node_zeroed_set(extent_node_t *node, bool zeroed); void extent_node_achunk_set(extent_node_t *node, bool achunk); void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx); +void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, + size_t size, bool zeroed); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) @@ -168,6 +163,20 @@ extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx) node->en_prof_tctx = tctx; } + +JEMALLOC_INLINE void +extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, + bool zeroed) +{ + + extent_node_arena_set(node, arena); + extent_node_addr_set(node, addr); + extent_node_size_set(node, size); + extent_node_zeroed_set(node, zeroed); + extent_node_achunk_set(node, false); + if (config_prof) + extent_node_prof_tctx_set(node, NULL); +} #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index a1d12cf6..8b55954c 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -182,6 +182,7 @@ extent_node_addr_get extent_node_addr_set extent_node_arena_get extent_node_arena_set +extent_node_init extent_node_prof_tctx_get extent_node_prof_tctx_set extent_node_size_get diff --git a/src/arena.c b/src/arena.c index a7a98e22..b068a4d8 100644 --- a/src/arena.c +++ b/src/arena.c @@ -465,9 +465,7 @@ arena_chunk_alloc_internal(arena_t *arena, bool *zero) chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL, chunksize, chunksize, zero); if (chunk != NULL) { - extent_node_arena_set(&chunk->node, arena); - extent_node_addr_set(&chunk->node, chunk); - extent_node_size_set(&chunk->node, chunksize); + extent_node_init(&chunk->node, arena, chunk, chunksize, *zero); extent_node_achunk_set(&chunk->node, true); if (chunk_register(chunk, &chunk->node)) { chunk_dalloc((void *)chunk, chunksize, arena->ind); @@ -1055,10 +1053,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, * just cached a node. */ assert(tnode != NULL); - extent_node_arena_set(tnode, arena); - extent_node_addr_set(tnode, addr); - extent_node_size_set(tnode, size); - extent_node_zeroed_set(tnode, zeroed); + extent_node_init(tnode, arena, addr, size, zeroed); arena_chunk_dirty_node_init(tnode); /* Stash. */ arena_chunk_dirty_insert(purge_runs_sentinel, diff --git a/src/base.c b/src/base.c index 819fa025..33e8b6f5 100644 --- a/src/base.c +++ b/src/base.c @@ -60,8 +60,7 @@ base_chunk_alloc(size_t minsize) if (config_stats) base_allocated += nsize; } - extent_node_addr_set(node, addr); - extent_node_size_set(node, csize); + extent_node_init(node, NULL, addr, csize, true); return (node); } @@ -84,8 +83,7 @@ base_alloc(size_t size) */ csize = CACHELINE_CEILING(size); - extent_node_addr_set(&key, NULL); - extent_node_size_set(&key, csize); + extent_node_init(&key, NULL, NULL, csize, false); malloc_mutex_lock(&base_mtx); node = extent_tree_szad_nsearch(&base_avail_szad, &key); if (node != NULL) { diff --git a/src/chunk.c b/src/chunk.c index 8bc87beb..59d72c9b 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -79,8 +79,7 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); - extent_node_addr_set(&key, new_addr); - extent_node_size_set(&key, alloc_size); + extent_node_init(&key, arena, new_addr, alloc_size, false); malloc_mutex_lock(&arena->chunks_mtx); node = (new_addr != NULL) ? extent_tree_ad_search(chunks_ad, &key) : extent_tree_szad_nsearch(chunks_szad, &key); @@ -121,9 +120,8 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, return (NULL); } } - extent_node_addr_set(node, (void *)((uintptr_t)(ret) + size)); - extent_node_size_set(node, trailsize); - extent_node_zeroed_set(node, zeroed); + extent_node_init(node, arena, (void *)((uintptr_t)(ret) + size), + trailsize, zeroed); extent_tree_szad_insert(chunks_szad, node); extent_tree_ad_insert(chunks_ad, node); arena_chunk_dirty_maybe_insert(arena, node, dirty); @@ -288,7 +286,8 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); malloc_mutex_lock(&arena->chunks_mtx); - extent_node_addr_set(&key, (void *)((uintptr_t)chunk + size)); + extent_node_init(&key, arena, (void *)((uintptr_t)chunk + size), 0, + false); node = extent_tree_ad_nsearch(chunks_ad, &key); /* Try to coalesce forward. */ if (node != NULL && extent_node_addr_get(node) == @@ -301,7 +300,7 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, extent_tree_szad_remove(chunks_szad, node); arena_chunk_dirty_maybe_remove(arena, node, dirty); extent_node_addr_set(node, chunk); - extent_node_size_set(node, extent_node_size_get(node) + size); + extent_node_size_set(node, size + extent_node_size_get(node)); extent_node_zeroed_set(node, extent_node_zeroed_get(node) && !unzeroed); extent_tree_szad_insert(chunks_szad, node); @@ -320,9 +319,7 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, pages_purge(chunk, size); goto label_return; } - extent_node_addr_set(node, chunk); - extent_node_size_set(node, size); - extent_node_zeroed_set(node, !unzeroed); + extent_node_init(node, arena, chunk, size, !unzeroed); extent_tree_ad_insert(chunks_ad, node); extent_tree_szad_insert(chunks_szad, node); arena_chunk_dirty_maybe_insert(arena, node, dirty); @@ -343,10 +340,10 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, extent_tree_szad_remove(chunks_szad, node); arena_chunk_dirty_maybe_remove(arena, node, dirty); extent_node_addr_set(node, extent_node_addr_get(prev)); - extent_node_size_set(node, extent_node_size_get(node) + - extent_node_size_get(prev)); - extent_node_zeroed_set(node, extent_node_zeroed_get(node) && - extent_node_zeroed_get(prev)); + extent_node_size_set(node, extent_node_size_get(prev) + + extent_node_size_get(node)); + extent_node_zeroed_set(node, extent_node_zeroed_get(prev) && + extent_node_zeroed_get(node)); extent_tree_szad_insert(chunks_szad, node); arena_chunk_dirty_maybe_insert(arena, node, dirty); diff --git a/src/huge.c b/src/huge.c index b9cae001..3092932e 100644 --- a/src/huge.c +++ b/src/huge.c @@ -73,11 +73,7 @@ huge_palloc(tsd_t *tsd, arena_t *arena, size_t usize, size_t alignment, return (NULL); } - extent_node_arena_set(node, arena); - extent_node_addr_set(node, ret); - extent_node_size_set(node, usize); - extent_node_achunk_set(node, false); - extent_node_zeroed_set(node, is_zeroed); + extent_node_init(node, arena, ret, usize, is_zeroed); if (huge_node_set(ret, node)) { arena_chunk_dalloc_huge(arena, ret, usize); From eafebfdfbe48bf8e95902d89cfa1eb3d5cd2fa5c Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 17 Feb 2015 16:12:31 -0800 Subject: [PATCH 236/352] Remove obsolete type arena_chunk_miscelms_t. --- include/jemalloc/internal/arena.h | 1 - 1 file changed, 1 deletion(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index f967be3a..0383f0c0 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -148,7 +148,6 @@ struct arena_chunk_map_misc_s { }; typedef rb_tree(arena_chunk_map_misc_t) arena_avail_tree_t; typedef rb_tree(arena_chunk_map_misc_t) arena_run_tree_t; -typedef qr(arena_chunk_map_misc_t) arena_chunk_miscelms_t; #endif /* JEMALLOC_ARENA_STRUCTS_A */ #ifdef JEMALLOC_ARENA_STRUCTS_B From 47701b22ee7c0df5e99efa0fcdcf98b9ff805b59 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 17 Feb 2015 22:23:10 -0800 Subject: [PATCH 237/352] arena_chunk_dirty_node_init() --> extent_node_dirty_linkage_init() --- include/jemalloc/internal/extent.h | 9 +++++++++ include/jemalloc/internal/private_symbols.txt | 1 + src/arena.c | 14 +++----------- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 9ee1b445..2f99debf 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -77,6 +77,7 @@ void extent_node_achunk_set(extent_node_t *node, bool achunk); void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx); void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, bool zeroed); +void extent_node_dirty_linkage_init(extent_node_t *node); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) @@ -177,6 +178,14 @@ extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, if (config_prof) extent_node_prof_tctx_set(node, NULL); } + +JEMALLOC_INLINE void +extent_node_dirty_linkage_init(extent_node_t *node) +{ + + qr_new(node, cd_link); + qr_new(&node->runs_dirty, rd_link); +} #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 8b55954c..0a8654b9 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -182,6 +182,7 @@ extent_node_addr_get extent_node_addr_set extent_node_arena_get extent_node_arena_set +extent_node_dirty_linkage_init extent_node_init extent_node_prof_tctx_get extent_node_prof_tctx_set diff --git a/src/arena.c b/src/arena.c index b068a4d8..205f598c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -152,14 +152,6 @@ arena_chunk_dirty_npages(const extent_node_t *node) return (extent_node_size_get(node) >> LG_PAGE); } -static void -arena_chunk_dirty_node_init(extent_node_t *node) -{ - - qr_new(node, cd_link); - qr_new(&node->runs_dirty, rd_link); -} - static void arena_chunk_dirty_insert(arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty, extent_node_t *node) @@ -181,8 +173,8 @@ void arena_chunk_dirty_maybe_insert(arena_t *arena, extent_node_t *node, bool dirty) { - arena_chunk_dirty_node_init(node); if (dirty) { + extent_node_dirty_linkage_init(node); arena_chunk_dirty_insert(&arena->runs_dirty, &arena->chunks_dirty, node); arena->ndirty += arena_chunk_dirty_npages(node); @@ -1054,7 +1046,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, */ assert(tnode != NULL); extent_node_init(tnode, arena, addr, size, zeroed); - arena_chunk_dirty_node_init(tnode); + extent_node_dirty_linkage_init(tnode); /* Stash. */ arena_chunk_dirty_insert(purge_runs_sentinel, purge_chunks_sentinel, tnode); @@ -1223,7 +1215,7 @@ arena_purge(arena_t *arena, bool all) npurge = arena_compute_npurge(arena, all); qr_new(&purge_runs_sentinel, rd_link); - arena_chunk_dirty_node_init(&purge_chunks_sentinel); + extent_node_dirty_linkage_init(&purge_chunks_sentinel); npurgeable = arena_stash_dirty(arena, all, npurge, &purge_runs_sentinel, &purge_chunks_sentinel); From 339c2b23b2d61993ac768afcc72af135662c6771 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 17 Feb 2015 22:25:56 -0800 Subject: [PATCH 238/352] Fix chunk_unmap() to propagate dirty state. Fix chunk_unmap() to propagate whether a chunk is dirty, and modify dirty chunk purging to record this information so it can be passed to chunk_unmap(). Since the broken version of chunk_unmap() claimed that all chunks were clean, this resulted in potential memory corruption for purging implementations that do not zero (e.g. MADV_FREE). This regression was introduced by ee41ad409a43d12900a5a3108f6c14f84e4eb0eb (Integrate whole chunks into unused dirty page purging machinery.). --- include/jemalloc/internal/chunk.h | 2 +- src/arena.c | 14 ++++++++++---- src/chunk.c | 6 +++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 96b9e159..8722dd0e 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -47,7 +47,7 @@ void *chunk_alloc_default(void *new_addr, size_t size, size_t alignment, void chunk_record(arena_t *arena, extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool dirty, void *chunk, size_t size); bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); -void chunk_unmap(arena_t *arena, void *chunk, size_t size); +void chunk_unmap(arena_t *arena, bool dirty, void *chunk, size_t size); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/src/arena.c b/src/arena.c index 205f598c..3d38386e 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1035,6 +1035,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, chunk = arena->chunk_alloc(addr, size, chunksize, &zero, arena->ind); assert(chunk == addr); + assert(zero == zeroed); /* * Create a temporary node to link into the ring of * stashed allocations. @@ -1075,7 +1076,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, /* Temporarily allocate the free dirty run. */ arena_run_split_large(arena, run, run_size, false); - /* Append to purge_runs for later processing. */ + /* Stash. */ if (false) qr_new(runselm, rd_link); /* Redundant. */ else { @@ -1114,9 +1115,12 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, if (runselm == &chunkselm->runs_dirty) { size_t size = extent_node_size_get(chunkselm); + bool unzeroed; - pages_purge(extent_node_addr_get(chunkselm), size); npages = size >> LG_PAGE; + unzeroed = pages_purge(extent_node_addr_get(chunkselm), + size); + extent_node_zeroed_set(chunkselm, !unzeroed); chunkselm = qr_next(chunkselm, cd_link); } else { arena_chunk_t *chunk; @@ -1180,11 +1184,13 @@ arena_unstash_purged(arena_t *arena, if (runselm == &chunkselm->runs_dirty) { extent_node_t *chunkselm_next = qr_next(chunkselm, cd_link); + bool dirty = !extent_node_zeroed_get(chunkselm); + void *addr = extent_node_addr_get(chunkselm); + size_t size = extent_node_size_get(chunkselm); arena_chunk_dirty_remove(chunkselm); - chunk_unmap(arena, extent_node_addr_get(chunkselm), - extent_node_size_get(chunkselm)); arena_node_dalloc(arena, chunkselm); chunkselm = chunkselm_next; + chunk_unmap(arena, dirty, addr, size); } else { arena_run_t *run = &runselm->run; qr_remove(runselm, rd_link); diff --git a/src/chunk.c b/src/chunk.c index 59d72c9b..774a978a 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -377,7 +377,7 @@ chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) } void -chunk_unmap(arena_t *arena, void *chunk, size_t size) +chunk_unmap(arena_t *arena, bool dirty, void *chunk, size_t size) { assert(chunk != NULL); @@ -387,10 +387,10 @@ chunk_unmap(arena_t *arena, void *chunk, size_t size) if (have_dss && chunk_in_dss(chunk)) { chunk_record(arena, &arena->chunks_szad_dss, - &arena->chunks_ad_dss, false, chunk, size); + &arena->chunks_ad_dss, dirty, chunk, size); } else if (chunk_dalloc_mmap(chunk, size)) { chunk_record(arena, &arena->chunks_szad_mmap, - &arena->chunks_ad_mmap, false, chunk, size); + &arena->chunks_ad_mmap, dirty, chunk, size); } } From 738e089a2e707dbfc70286f7deeebc68e03d2347 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 18 Feb 2015 01:15:50 -0800 Subject: [PATCH 239/352] Rename "dirty chunks" to "cached chunks". Rename "dirty chunks" to "cached chunks", in order to avoid overloading the term "dirty". Fix the regression caused by 339c2b23b2d61993ac768afcc72af135662c6771 (Fix chunk_unmap() to propagate dirty state.), and actually address what that change attempted, which is to only purge chunks once, and propagate whether zeroed pages resulted into chunk_record(). --- include/jemalloc/internal/arena.h | 22 +++--- include/jemalloc/internal/chunk.h | 5 +- include/jemalloc/internal/extent.h | 27 ++++++- include/jemalloc/internal/private_symbols.txt | 6 +- src/arena.c | 70 +++++++------------ src/chunk.c | 45 ++++++------ src/chunk_dss.c | 2 +- 7 files changed, 91 insertions(+), 86 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 0383f0c0..3d79c627 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -318,14 +318,14 @@ struct arena_s { /* * Unused dirty memory this arena manages. Dirty memory is conceptually - * tracked as an arbitrarily interleaved LRU of runs and chunks, but the - * list linkage is actually semi-duplicated in order to avoid extra - * arena_chunk_map_misc_t space overhead. + * tracked as an arbitrarily interleaved LRU of dirty runs and cached + * chunks, but the list linkage is actually semi-duplicated in order to + * avoid extra arena_chunk_map_misc_t space overhead. * * LRU-----------------------------------------------------------MRU * * ______________ ___ ___ - * ...-->|chunks_dirty|<--------->|c|<-------------------->|c|<--... + * ...-->|chunks_cache|<--------->|c|<-------------------->|c|<--... * -------------- |h| |h| * ____________ _____ |u| _____ _____ |u| * ...-->|runs_dirty|<-->|run|<-->|n|<-->|run|<-->|run|<-->|n|<--... @@ -333,7 +333,7 @@ struct arena_s { * --- --- */ arena_chunk_map_misc_t runs_dirty; - extent_node_t chunks_dirty; + extent_node_t chunks_cache; /* Extant huge allocations. */ ql_head(extent_node_t) huge; @@ -347,8 +347,8 @@ struct arena_s { * orderings are needed, which is why there are two trees with the same * contents. */ - extent_tree_t chunks_szad_dirty; - extent_tree_t chunks_ad_dirty; + extent_tree_t chunks_szad_cache; + extent_tree_t chunks_ad_cache; extent_tree_t chunks_szad_mmap; extent_tree_t chunks_ad_mmap; extent_tree_t chunks_szad_dss; @@ -384,10 +384,10 @@ extern size_t arena_maxclass; /* Max size class for arenas. */ extern unsigned nlclasses; /* Number of large size classes. */ extern unsigned nhclasses; /* Number of huge size classes. */ -void arena_chunk_dirty_maybe_insert(arena_t *arena, extent_node_t *node, - bool dirty); -void arena_chunk_dirty_maybe_remove(arena_t *arena, extent_node_t *node, - bool dirty); +void arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, + bool cache); +void arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, + bool cache); extent_node_t *arena_node_alloc(arena_t *arena); void arena_node_dalloc(arena_t *arena, extent_node_t *node); void *arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 8722dd0e..bf6acbd1 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -45,9 +45,10 @@ void *chunk_alloc_arena(chunk_alloc_t *chunk_alloc, void *chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind); void chunk_record(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, bool dirty, void *chunk, size_t size); + extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, + bool zeroed); bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); -void chunk_unmap(arena_t *arena, bool dirty, void *chunk, size_t size); +void chunk_unmap(arena_t *arena, void *chunk, size_t size, bool zeroed); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 2f99debf..81ff40b8 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -33,9 +33,9 @@ struct extent_node_s { /* Profile counters, used for huge objects. */ prof_tctx_t *en_prof_tctx; - /* Linkage for arena's runs_dirty and chunks_dirty rings. */ - qr(extent_node_t) cd_link; + /* Linkage for arena's runs_dirty and chunks_cache rings. */ arena_chunk_map_misc_t runs_dirty; + qr(extent_node_t) cc_link; union { /* Linkage for the size/address-ordered tree. */ @@ -78,6 +78,9 @@ void extent_node_prof_tctx_set(extent_node_t *node, prof_tctx_t *tctx); void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, bool zeroed); void extent_node_dirty_linkage_init(extent_node_t *node); +void extent_node_dirty_insert(extent_node_t *node, + arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty); +void extent_node_dirty_remove(extent_node_t *node); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_EXTENT_C_)) @@ -183,9 +186,27 @@ JEMALLOC_INLINE void extent_node_dirty_linkage_init(extent_node_t *node) { - qr_new(node, cd_link); qr_new(&node->runs_dirty, rd_link); + qr_new(node, cc_link); } + +JEMALLOC_INLINE void +extent_node_dirty_insert(extent_node_t *node, + arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty) +{ + + qr_meld(runs_dirty, &node->runs_dirty, rd_link); + qr_meld(chunks_dirty, node, cc_link); +} + +JEMALLOC_INLINE void +extent_node_dirty_remove(extent_node_t *node) +{ + + qr_remove(&node->runs_dirty, rd_link); + qr_remove(node, cc_link); +} + #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 0a8654b9..dfe62ce5 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -12,9 +12,9 @@ arena_boot arena_choose arena_choose_hard arena_chunk_alloc_huge +arena_chunk_cache_maybe_insert +arena_chunk_cache_maybe_remove arena_chunk_dalloc_huge -arena_chunk_dirty_maybe_insert -arena_chunk_dirty_maybe_remove arena_chunk_ralloc_huge_expand arena_chunk_ralloc_huge_shrink arena_chunk_ralloc_huge_similar @@ -182,7 +182,9 @@ extent_node_addr_get extent_node_addr_set extent_node_arena_get extent_node_arena_set +extent_node_dirty_insert extent_node_dirty_linkage_init +extent_node_dirty_remove extent_node_init extent_node_prof_tctx_get extent_node_prof_tctx_set diff --git a/src/arena.c b/src/arena.c index 3d38386e..762b8182 100644 --- a/src/arena.c +++ b/src/arena.c @@ -152,41 +152,24 @@ arena_chunk_dirty_npages(const extent_node_t *node) return (extent_node_size_get(node) >> LG_PAGE); } -static void -arena_chunk_dirty_insert(arena_chunk_map_misc_t *runs_dirty, - extent_node_t *chunks_dirty, extent_node_t *node) -{ - - qr_meld(chunks_dirty, node, cd_link); - qr_meld(runs_dirty, &node->runs_dirty, rd_link); -} - -static void -arena_chunk_dirty_remove(extent_node_t *node) -{ - - qr_remove(node, cd_link); - qr_remove(&node->runs_dirty, rd_link); -} - void -arena_chunk_dirty_maybe_insert(arena_t *arena, extent_node_t *node, bool dirty) +arena_chunk_cache_maybe_insert(arena_t *arena, extent_node_t *node, bool cache) { - if (dirty) { + if (cache) { extent_node_dirty_linkage_init(node); - arena_chunk_dirty_insert(&arena->runs_dirty, - &arena->chunks_dirty, node); + extent_node_dirty_insert(node, &arena->runs_dirty, + &arena->chunks_cache); arena->ndirty += arena_chunk_dirty_npages(node); } } void -arena_chunk_dirty_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) +arena_chunk_cache_maybe_remove(arena_t *arena, extent_node_t *node, bool dirty) { if (dirty) { - arena_chunk_dirty_remove(node); + extent_node_dirty_remove(node); assert(arena->ndirty >= arena_chunk_dirty_npages(node)); arena->ndirty -= arena_chunk_dirty_npages(node); } @@ -954,14 +937,14 @@ arena_dirty_count(arena_t *arena) extent_node_t *chunkselm; for (runselm = qr_next(&arena->runs_dirty, rd_link), - chunkselm = qr_next(&arena->chunks_dirty, cd_link); + chunkselm = qr_next(&arena->chunks_cache, cc_link); runselm != &arena->runs_dirty; runselm = qr_next(runselm, rd_link)) { size_t npages; if (runselm == &chunkselm->runs_dirty) { npages = extent_node_size_get(chunkselm) >> LG_PAGE; - chunkselm = qr_next(chunkselm, cd_link); + chunkselm = qr_next(chunkselm, cc_link); } else { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); @@ -1010,7 +993,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, /* Stash at least npurge pages. */ for (runselm = qr_next(&arena->runs_dirty, rd_link), - chunkselm = qr_next(&arena->chunks_dirty, cd_link); + chunkselm = qr_next(&arena->chunks_cache, cc_link); runselm != &arena->runs_dirty; runselm = runselm_next) { size_t npages; runselm_next = qr_next(runselm, rd_link); @@ -1022,7 +1005,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, bool zeroed, zero; UNUSED void *chunk; - chunkselm_next = qr_next(chunkselm, cd_link); + chunkselm_next = qr_next(chunkselm, cc_link); /* * Cache contents of chunkselm prior to it being * destroyed as a side effect of allocating the chunk. @@ -1038,19 +1021,16 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, assert(zero == zeroed); /* * Create a temporary node to link into the ring of - * stashed allocations. + * stashed allocations. OOM shouldn't be possible + * because chunk allocation just cached a node. */ tnode = arena_node_alloc(arena); - /* - * OOM shouldn't be possible because chunk allocation - * just cached a node. - */ assert(tnode != NULL); + /* Stash. */ extent_node_init(tnode, arena, addr, size, zeroed); extent_node_dirty_linkage_init(tnode); - /* Stash. */ - arena_chunk_dirty_insert(purge_runs_sentinel, - purge_chunks_sentinel, tnode); + extent_node_dirty_insert(tnode, purge_runs_sentinel, + purge_chunks_sentinel); npages = size >> LG_PAGE; chunkselm = chunkselm_next; } else { @@ -1108,7 +1088,7 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, malloc_mutex_unlock(&arena->lock); for (runselm = qr_next(purge_runs_sentinel, rd_link), - chunkselm = qr_next(purge_chunks_sentinel, cd_link); + chunkselm = qr_next(purge_chunks_sentinel, cc_link); runselm != purge_runs_sentinel; runselm = qr_next(runselm, rd_link)) { size_t npages; @@ -1121,7 +1101,7 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, unzeroed = pages_purge(extent_node_addr_get(chunkselm), size); extent_node_zeroed_set(chunkselm, !unzeroed); - chunkselm = qr_next(chunkselm, cd_link); + chunkselm = qr_next(chunkselm, cc_link); } else { arena_chunk_t *chunk; size_t pageind, run_size, flag_unzeroed, i; @@ -1178,19 +1158,19 @@ arena_unstash_purged(arena_t *arena, /* Deallocate runs. */ for (runselm = qr_next(purge_runs_sentinel, rd_link), - chunkselm = qr_next(purge_chunks_sentinel, cd_link); + chunkselm = qr_next(purge_chunks_sentinel, cc_link); runselm != purge_runs_sentinel; runselm = runselm_next) { runselm_next = qr_next(runselm, rd_link); if (runselm == &chunkselm->runs_dirty) { extent_node_t *chunkselm_next = qr_next(chunkselm, - cd_link); - bool dirty = !extent_node_zeroed_get(chunkselm); + cc_link); void *addr = extent_node_addr_get(chunkselm); size_t size = extent_node_size_get(chunkselm); - arena_chunk_dirty_remove(chunkselm); + bool zeroed = extent_node_zeroed_get(chunkselm); + extent_node_dirty_remove(chunkselm); arena_node_dalloc(arena, chunkselm); chunkselm = chunkselm_next; - chunk_unmap(arena, dirty, addr, size); + chunk_unmap(arena, addr, size, zeroed); } else { arena_run_t *run = &runselm->run; qr_remove(runselm, rd_link); @@ -2612,14 +2592,14 @@ arena_new(unsigned ind) arena_avail_tree_new(&arena->runs_avail); qr_new(&arena->runs_dirty, rd_link); - qr_new(&arena->chunks_dirty, cd_link); + qr_new(&arena->chunks_cache, cc_link); ql_new(&arena->huge); if (malloc_mutex_init(&arena->huge_mtx)) return (NULL); - extent_tree_szad_new(&arena->chunks_szad_dirty); - extent_tree_ad_new(&arena->chunks_ad_dirty); + extent_tree_szad_new(&arena->chunks_szad_cache); + extent_tree_ad_new(&arena->chunks_ad_cache); extent_tree_szad_new(&arena->chunks_szad_mmap); extent_tree_ad_new(&arena->chunks_ad_mmap); extent_tree_szad_new(&arena->chunks_szad_dss); diff --git a/src/chunk.c b/src/chunk.c index 774a978a..264e4f27 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -64,7 +64,7 @@ chunk_deregister(const void *chunk, const extent_node_t *node) static void * chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, bool dirty, void *new_addr, size_t size, + extent_tree_t *chunks_ad, bool cache, void *new_addr, size_t size, size_t alignment, bool *zero) { void *ret; @@ -100,13 +100,13 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, /* Remove node from the tree. */ extent_tree_szad_remove(chunks_szad, node); extent_tree_ad_remove(chunks_ad, node); - arena_chunk_dirty_maybe_remove(arena, node, dirty); + arena_chunk_cache_maybe_remove(arena, node, cache); if (leadsize != 0) { /* Insert the leading space as a smaller chunk. */ extent_node_size_set(node, leadsize); extent_tree_szad_insert(chunks_szad, node); extent_tree_ad_insert(chunks_ad, node); - arena_chunk_dirty_maybe_insert(arena, node, dirty); + arena_chunk_cache_maybe_insert(arena, node, cache); node = NULL; } if (trailsize != 0) { @@ -116,7 +116,7 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, if (node == NULL) { malloc_mutex_unlock(&arena->chunks_mtx); chunk_record(arena, chunks_szad, chunks_ad, - dirty, ret, size); + cache, ret, size, zeroed); return (NULL); } } @@ -124,7 +124,7 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, trailsize, zeroed); extent_tree_szad_insert(chunks_szad, node); extent_tree_ad_insert(chunks_ad, node); - arena_chunk_dirty_maybe_insert(arena, node, dirty); + arena_chunk_cache_maybe_insert(arena, node, cache); node = NULL; } malloc_mutex_unlock(&arena->chunks_mtx); @@ -177,9 +177,9 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment, assert(alignment != 0); assert((alignment & chunksize_mask) == 0); - /* dirty. */ - if ((ret = chunk_recycle(arena, &arena->chunks_szad_dirty, - &arena->chunks_ad_dirty, true, new_addr, size, alignment, zero)) != + /* cache. */ + if ((ret = chunk_recycle(arena, &arena->chunks_szad_cache, + &arena->chunks_ad_cache, true, new_addr, size, alignment, zero)) != NULL) return (ret); /* "primary" dss. */ @@ -276,13 +276,14 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, void chunk_record(arena_t *arena, extent_tree_t *chunks_szad, - extent_tree_t *chunks_ad, bool dirty, void *chunk, size_t size) + extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, bool zeroed) { bool unzeroed; extent_node_t *node, *prev; extent_node_t key; - unzeroed = dirty ? true : pages_purge(chunk, size); + assert(!cache || !zeroed); + unzeroed = cache || !zeroed; JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); malloc_mutex_lock(&arena->chunks_mtx); @@ -298,13 +299,13 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, * remove/insert from/into chunks_szad. */ extent_tree_szad_remove(chunks_szad, node); - arena_chunk_dirty_maybe_remove(arena, node, dirty); + arena_chunk_cache_maybe_remove(arena, node, cache); extent_node_addr_set(node, chunk); extent_node_size_set(node, size + extent_node_size_get(node)); extent_node_zeroed_set(node, extent_node_zeroed_get(node) && !unzeroed); extent_tree_szad_insert(chunks_szad, node); - arena_chunk_dirty_maybe_insert(arena, node, dirty); + arena_chunk_cache_maybe_insert(arena, node, cache); } else { /* Coalescing forward failed, so insert a new node. */ node = arena_node_alloc(arena); @@ -315,14 +316,14 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, * pages have already been purged, so that this is only * a virtual memory leak. */ - if (dirty) + if (cache) pages_purge(chunk, size); goto label_return; } extent_node_init(node, arena, chunk, size, !unzeroed); extent_tree_ad_insert(chunks_ad, node); extent_tree_szad_insert(chunks_szad, node); - arena_chunk_dirty_maybe_insert(arena, node, dirty); + arena_chunk_cache_maybe_insert(arena, node, cache); } /* Try to coalesce backward. */ @@ -336,16 +337,16 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, */ extent_tree_szad_remove(chunks_szad, prev); extent_tree_ad_remove(chunks_ad, prev); - arena_chunk_dirty_maybe_remove(arena, prev, dirty); + arena_chunk_cache_maybe_remove(arena, prev, cache); extent_tree_szad_remove(chunks_szad, node); - arena_chunk_dirty_maybe_remove(arena, node, dirty); + arena_chunk_cache_maybe_remove(arena, node, cache); extent_node_addr_set(node, extent_node_addr_get(prev)); extent_node_size_set(node, extent_node_size_get(prev) + extent_node_size_get(node)); extent_node_zeroed_set(node, extent_node_zeroed_get(prev) && extent_node_zeroed_get(node)); extent_tree_szad_insert(chunks_szad, node); - arena_chunk_dirty_maybe_insert(arena, node, dirty); + arena_chunk_cache_maybe_insert(arena, node, cache); arena_node_dalloc(arena, prev); } @@ -363,8 +364,8 @@ chunk_cache(arena_t *arena, void *chunk, size_t size) assert(size != 0); assert((size & chunksize_mask) == 0); - chunk_record(arena, &arena->chunks_szad_dirty, &arena->chunks_ad_dirty, - true, chunk, size); + chunk_record(arena, &arena->chunks_szad_cache, &arena->chunks_ad_cache, + true, chunk, size, false); } /* Default arena chunk deallocation routine in the absence of user override. */ @@ -377,7 +378,7 @@ chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) } void -chunk_unmap(arena_t *arena, bool dirty, void *chunk, size_t size) +chunk_unmap(arena_t *arena, void *chunk, size_t size, bool zeroed) { assert(chunk != NULL); @@ -387,10 +388,10 @@ chunk_unmap(arena_t *arena, bool dirty, void *chunk, size_t size) if (have_dss && chunk_in_dss(chunk)) { chunk_record(arena, &arena->chunks_szad_dss, - &arena->chunks_ad_dss, dirty, chunk, size); + &arena->chunks_ad_dss, false, chunk, size, zeroed); } else if (chunk_dalloc_mmap(chunk, size)) { chunk_record(arena, &arena->chunks_szad_mmap, - &arena->chunks_ad_mmap, dirty, chunk, size); + &arena->chunks_ad_mmap, false, chunk, size, zeroed); } } diff --git a/src/chunk_dss.c b/src/chunk_dss.c index c3c48481..6fbe31bf 100644 --- a/src/chunk_dss.c +++ b/src/chunk_dss.c @@ -137,7 +137,7 @@ chunk_alloc_dss(arena_t *arena, void *new_addr, size_t size, size_t alignment, chunk_record(arena, &arena->chunks_szad_dss, &arena->chunks_ad_dss, false, cpad, - cpad_size); + cpad_size, false); } if (*zero) { JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED( From 99bd94fb65a0b6423c4efcc3e3e501179b92a4db Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 18 Feb 2015 16:40:53 -0800 Subject: [PATCH 240/352] Fix chunk cache races. These regressions were introduced by ee41ad409a43d12900a5a3108f6c14f84e4eb0eb (Integrate whole chunks into unused dirty page purging machinery.). --- include/jemalloc/internal/arena.h | 1 + include/jemalloc/internal/chunk.h | 13 +- include/jemalloc/internal/private_symbols.txt | 8 +- src/arena.c | 266 +++++++++++------- src/chunk.c | 114 +++++--- 5 files changed, 263 insertions(+), 139 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 3d79c627..42086ca1 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -399,6 +399,7 @@ void arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, size_t usize); bool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, size_t usize, bool *zero); +void arena_maybe_purge(arena_t *arena); void arena_purge_all(arena_t *arena); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, index_t binind, uint64_t prof_accumbytes); diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index bf6acbd1..1a968a58 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -39,16 +39,21 @@ extern size_t chunk_npages; bool chunk_register(const void *chunk, const extent_node_t *node); void chunk_deregister(const void *chunk, const extent_node_t *node); void *chunk_alloc_base(size_t size); -void *chunk_alloc_arena(chunk_alloc_t *chunk_alloc, - chunk_dalloc_t *chunk_dalloc, unsigned arena_ind, void *new_addr, - size_t size, size_t alignment, bool *zero); +void *chunk_alloc_cache(arena_t *arena, void *new_addr, size_t size, + size_t alignment, bool *zero, bool dalloc_node); void *chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind); +void *chunk_alloc_wrapper(arena_t *arena, chunk_alloc_t *chunk_alloc, + void *new_addr, size_t size, size_t alignment, bool *zero); void chunk_record(arena_t *arena, extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, void *chunk, size_t size, bool zeroed); +void chunk_dalloc_cache(arena_t *arena, void *chunk, size_t size); +void chunk_dalloc_arena(arena_t *arena, void *chunk, size_t size, + bool zeroed); bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); -void chunk_unmap(arena_t *arena, void *chunk, size_t size, bool zeroed); +void chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc, + void *chunk, size_t size); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index dfe62ce5..7c217c74 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -53,6 +53,7 @@ arena_mapbitsp_read arena_mapbitsp_write arena_maxclass arena_maxrun +arena_maybe_purge arena_metadata_allocated_add arena_metadata_allocated_get arena_metadata_allocated_sub @@ -124,14 +125,18 @@ bootstrap_free bootstrap_malloc bt_init buferror -chunk_alloc_arena +chunk_alloc_cache chunk_alloc_base chunk_alloc_default chunk_alloc_dss chunk_alloc_mmap +chunk_alloc_wrapper chunk_boot +chunk_dalloc_arena +chunk_dalloc_cache chunk_dalloc_default chunk_dalloc_mmap +chunk_dalloc_wrapper chunk_deregister chunk_dss_boot chunk_dss_postfork_child @@ -147,7 +152,6 @@ chunk_postfork_parent chunk_prefork chunk_record chunk_register -chunk_unmap chunks_rtree chunksize chunksize_mask diff --git a/src/arena.c b/src/arena.c index 762b8182..78aa1ae0 100644 --- a/src/arena.c +++ b/src/arena.c @@ -20,7 +20,6 @@ unsigned nhclasses; /* Number of huge size classes. */ * definition. */ -static void arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk); static void arena_purge(arena_t *arena, bool all); static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned); @@ -427,27 +426,53 @@ arena_chunk_init_spare(arena_t *arena) return (chunk); } +static bool +arena_chunk_register(arena_t *arena, arena_chunk_t *chunk, bool zero) +{ + + extent_node_init(&chunk->node, arena, chunk, chunksize, zero); + extent_node_achunk_set(&chunk->node, true); + return (chunk_register(chunk, &chunk->node)); +} + +static arena_chunk_t * +arena_chunk_alloc_internal_hard(arena_t *arena, bool *zero) +{ + arena_chunk_t *chunk; + chunk_alloc_t *chunk_alloc = arena->chunk_alloc; + chunk_dalloc_t *chunk_dalloc = arena->chunk_dalloc; + + malloc_mutex_unlock(&arena->lock); + chunk = (arena_chunk_t *)chunk_alloc_wrapper(arena, chunk_alloc, NULL, + chunksize, chunksize, zero); + if (chunk != NULL && arena_chunk_register(arena, chunk, *zero)) { + chunk_dalloc_wrapper(arena, chunk_dalloc, (void *)chunk, + chunksize); + chunk = NULL; + } + malloc_mutex_lock(&arena->lock); + + return (chunk); +} + static arena_chunk_t * arena_chunk_alloc_internal(arena_t *arena, bool *zero) { arena_chunk_t *chunk; - chunk_alloc_t *chunk_alloc; - chunk_dalloc_t *chunk_dalloc; - chunk_alloc = arena->chunk_alloc; - chunk_dalloc = arena->chunk_dalloc; - malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc_arena(chunk_alloc, chunk_dalloc, - arena->ind, NULL, chunksize, chunksize, zero); - if (chunk != NULL) { - extent_node_init(&chunk->node, arena, chunk, chunksize, *zero); - extent_node_achunk_set(&chunk->node, true); - if (chunk_register(chunk, &chunk->node)) { - chunk_dalloc((void *)chunk, chunksize, arena->ind); - chunk = NULL; + if (likely(arena->chunk_alloc == chunk_alloc_default)) { + chunk = chunk_alloc_cache(arena, NULL, chunksize, chunksize, + zero, true); + if (chunk != NULL && arena_chunk_register(arena, chunk, + *zero)) { + chunk_dalloc_cache(arena, chunk, chunksize); + return (NULL); } - } - malloc_mutex_lock(&arena->lock); + } else + chunk = NULL; + if (chunk == NULL) + chunk = arena_chunk_alloc_internal_hard(arena, zero); + if (config_stats && chunk != NULL) { arena->stats.mapped += chunksize; arena->stats.metadata_mapped += (map_bias << LG_PAGE); @@ -553,11 +578,19 @@ arena_chunk_dalloc(arena_t *arena, arena_chunk_t *chunk) arena_run_dirty_remove(arena, spare, map_bias, chunk_npages-map_bias); } - chunk_dalloc = arena->chunk_dalloc; - malloc_mutex_unlock(&arena->lock); + chunk_deregister(spare, &spare->node); - chunk_dalloc((void *)spare, chunksize, arena->ind); - malloc_mutex_lock(&arena->lock); + + chunk_dalloc = arena->chunk_dalloc; + if (likely(chunk_dalloc == chunk_dalloc_default)) + chunk_dalloc_cache(arena, (void *)spare, chunksize); + else { + malloc_mutex_unlock(&arena->lock); + chunk_dalloc_wrapper(arena, chunk_dalloc, (void *)spare, + chunksize); + malloc_mutex_lock(&arena->lock); + } + if (config_stats) { arena->stats.mapped -= chunksize; arena->stats.metadata_mapped -= (map_bias << LG_PAGE); @@ -661,28 +694,14 @@ arena_node_dalloc(arena_t *arena, extent_node_t *node) malloc_mutex_unlock(&arena->node_cache_mtx); } -void * -arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, - bool *zero) +static void * +arena_chunk_alloc_huge_hard(arena_t *arena, chunk_alloc_t *chunk_alloc, + size_t usize, size_t alignment, bool *zero, size_t csize) { void *ret; - chunk_alloc_t *chunk_alloc; - chunk_dalloc_t *chunk_dalloc; - size_t csize = CHUNK_CEILING(usize); - malloc_mutex_lock(&arena->lock); - chunk_alloc = arena->chunk_alloc; - chunk_dalloc = arena->chunk_dalloc; - if (config_stats) { - /* Optimistically update stats prior to unlocking. */ - arena_huge_malloc_stats_update(arena, usize); - arena->stats.mapped += usize; - } - arena->nactive += (usize >> LG_PAGE); - malloc_mutex_unlock(&arena->lock); - - ret = chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, NULL, - csize, alignment, zero); + ret = chunk_alloc_wrapper(arena, chunk_alloc, NULL, csize, alignment, + zero); if (ret == NULL) { /* Revert optimistic stats updates. */ malloc_mutex_lock(&arena->lock); @@ -692,12 +711,42 @@ arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, } arena->nactive -= (usize >> LG_PAGE); malloc_mutex_unlock(&arena->lock); - return (NULL); } - if (config_stats) - stats_cactive_add(usize); + return (ret); +} +void * +arena_chunk_alloc_huge(arena_t *arena, size_t usize, size_t alignment, + bool *zero) +{ + void *ret; + chunk_alloc_t *chunk_alloc; + size_t csize = CHUNK_CEILING(usize); + + malloc_mutex_lock(&arena->lock); + + /* Optimistically update stats. */ + if (config_stats) { + arena_huge_malloc_stats_update(arena, usize); + arena->stats.mapped += usize; + } + arena->nactive += (usize >> LG_PAGE); + + chunk_alloc = arena->chunk_alloc; + if (likely(chunk_alloc == chunk_alloc_default)) { + ret = chunk_alloc_cache(arena, NULL, csize, alignment, zero, + true); + } else + ret = NULL; + malloc_mutex_unlock(&arena->lock); + if (ret == NULL) { + ret = arena_chunk_alloc_huge_hard(arena, chunk_alloc, usize, + alignment, zero, csize); + } + + if (config_stats && ret != NULL) + stats_cactive_add(usize); return (ret); } @@ -705,7 +754,9 @@ void arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) { chunk_dalloc_t *chunk_dalloc; + size_t csize; + csize = CHUNK_CEILING(usize); malloc_mutex_lock(&arena->lock); chunk_dalloc = arena->chunk_dalloc; if (config_stats) { @@ -714,8 +765,14 @@ arena_chunk_dalloc_huge(arena_t *arena, void *chunk, size_t usize) stats_cactive_sub(usize); } arena->nactive -= (usize >> LG_PAGE); - malloc_mutex_unlock(&arena->lock); - chunk_dalloc(chunk, CHUNK_CEILING(usize), arena->ind); + + if (likely(chunk_dalloc == chunk_dalloc_default)) { + chunk_dalloc_cache(arena, chunk, csize); + malloc_mutex_unlock(&arena->lock); + } else { + malloc_mutex_unlock(&arena->lock); + chunk_dalloc_wrapper(arena, chunk_dalloc, chunk, csize); + } } void @@ -747,12 +804,10 @@ void arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, size_t usize) { - chunk_dalloc_t *chunk_dalloc; size_t udiff = oldsize - usize; size_t cdiff = CHUNK_CEILING(oldsize) - CHUNK_CEILING(usize); malloc_mutex_lock(&arena->lock); - chunk_dalloc = arena->chunk_dalloc; if (config_stats) { arena_huge_ralloc_stats_update(arena, oldsize, usize); if (cdiff != 0) { @@ -761,52 +816,81 @@ arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, } } arena->nactive -= udiff >> LG_PAGE; - malloc_mutex_unlock(&arena->lock); + if (cdiff != 0) { - chunk_dalloc((void *)((uintptr_t)chunk + CHUNK_CEILING(usize)), - cdiff, arena->ind); + chunk_dalloc_t *chunk_dalloc = arena->chunk_dalloc; + void *nchunk = (void *)((uintptr_t)chunk + + CHUNK_CEILING(usize)); + + if (likely(chunk_dalloc == chunk_dalloc_default)) { + chunk_dalloc_cache(arena, nchunk, cdiff); + malloc_mutex_unlock(&arena->lock); + } else { + malloc_mutex_unlock(&arena->lock); + chunk_dalloc_wrapper(arena, chunk_dalloc, nchunk, + cdiff); + } + } else + malloc_mutex_unlock(&arena->lock); +} + +bool +arena_chunk_ralloc_huge_expand_hard(arena_t *arena, chunk_alloc_t *chunk_alloc, + size_t oldsize, size_t usize, bool *zero, void *nchunk, size_t udiff, + size_t cdiff) +{ + bool err; + + err = (chunk_alloc_wrapper(arena, chunk_alloc, nchunk, cdiff, chunksize, + zero) == NULL); + if (err) { + /* Revert optimistic stats updates. */ + malloc_mutex_lock(&arena->lock); + if (config_stats) { + arena_huge_ralloc_stats_update_undo(arena, oldsize, + usize); + arena->stats.mapped -= cdiff; + } + arena->nactive -= (udiff >> LG_PAGE); + malloc_mutex_unlock(&arena->lock); } + return (err); } bool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, size_t usize, bool *zero) { + bool err; chunk_alloc_t *chunk_alloc; - chunk_dalloc_t *chunk_dalloc; + void *nchunk = (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)); size_t udiff = usize - oldsize; size_t cdiff = CHUNK_CEILING(usize) - CHUNK_CEILING(oldsize); malloc_mutex_lock(&arena->lock); - chunk_alloc = arena->chunk_alloc; - chunk_dalloc = arena->chunk_dalloc; + + /* Optimistically update stats. */ if (config_stats) { - /* Optimistically update stats prior to unlocking. */ arena_huge_ralloc_stats_update(arena, oldsize, usize); arena->stats.mapped += cdiff; } arena->nactive += (udiff >> LG_PAGE); - malloc_mutex_unlock(&arena->lock); - if (chunk_alloc_arena(chunk_alloc, chunk_dalloc, arena->ind, - (void *)((uintptr_t)chunk + CHUNK_CEILING(oldsize)), cdiff, - chunksize, zero) == NULL) { - /* Revert optimistic stats updates. */ - malloc_mutex_lock(&arena->lock); - if (config_stats) { - arena_huge_ralloc_stats_update_undo(arena, - oldsize, usize); - arena->stats.mapped -= cdiff; - } - arena->nactive -= (udiff >> LG_PAGE); - malloc_mutex_unlock(&arena->lock); - return (true); + chunk_alloc = arena->chunk_alloc; + if (likely(chunk_alloc == chunk_alloc_default)) { + err = (chunk_alloc_cache(arena, nchunk, cdiff, chunksize, zero, + true) == NULL); + } else + err = true; + malloc_mutex_unlock(&arena->lock); + if (err) { + err = arena_chunk_ralloc_huge_expand_hard(arena, chunk_alloc, + oldsize, usize, zero, nchunk, udiff, cdiff); } - if (config_stats) + if (config_stats && !err) stats_cactive_add(udiff); - - return (false); + return (err); } static arena_run_t * @@ -909,7 +993,7 @@ arena_run_alloc_small(arena_t *arena, size_t size, index_t binind) return (arena_run_alloc_small_helper(arena, size, binind)); } -JEMALLOC_INLINE_C void +void arena_maybe_purge(arena_t *arena) { size_t threshold; @@ -999,39 +1083,25 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, runselm_next = qr_next(runselm, rd_link); if (runselm == &chunkselm->runs_dirty) { - extent_node_t *chunkselm_next, *tnode; - void *addr; - size_t size; - bool zeroed, zero; + extent_node_t *chunkselm_next; + bool zero; UNUSED void *chunk; chunkselm_next = qr_next(chunkselm, cc_link); /* - * Cache contents of chunkselm prior to it being - * destroyed as a side effect of allocating the chunk. + * Allocate. chunkselm remains valid due to the + * dalloc_node=false argument to chunk_alloc_cache(). */ - addr = extent_node_addr_get(chunkselm); - size = extent_node_size_get(chunkselm); - zeroed = extent_node_zeroed_get(chunkselm); - /* Allocate. */ zero = false; - chunk = arena->chunk_alloc(addr, size, chunksize, &zero, - arena->ind); - assert(chunk == addr); - assert(zero == zeroed); - /* - * Create a temporary node to link into the ring of - * stashed allocations. OOM shouldn't be possible - * because chunk allocation just cached a node. - */ - tnode = arena_node_alloc(arena); - assert(tnode != NULL); - /* Stash. */ - extent_node_init(tnode, arena, addr, size, zeroed); - extent_node_dirty_linkage_init(tnode); - extent_node_dirty_insert(tnode, purge_runs_sentinel, + chunk = chunk_alloc_cache(arena, + extent_node_addr_get(chunkselm), + extent_node_size_get(chunkselm), chunksize, &zero, + false); + assert(chunk == extent_node_addr_get(chunkselm)); + assert(zero == extent_node_zeroed_get(chunkselm)); + extent_node_dirty_insert(chunkselm, purge_runs_sentinel, purge_chunks_sentinel); - npages = size >> LG_PAGE; + npages = extent_node_size_get(chunkselm) >> LG_PAGE; chunkselm = chunkselm_next; } else { arena_chunk_t *chunk = @@ -1170,7 +1240,7 @@ arena_unstash_purged(arena_t *arena, extent_node_dirty_remove(chunkselm); arena_node_dalloc(arena, chunkselm); chunkselm = chunkselm_next; - chunk_unmap(arena, addr, size, zeroed); + chunk_dalloc_arena(arena, addr, size, zeroed); } else { arena_run_t *run = &runselm->run; qr_remove(runselm, rd_link); diff --git a/src/chunk.c b/src/chunk.c index 264e4f27..08f21f6a 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -65,7 +65,7 @@ chunk_deregister(const void *chunk, const extent_node_t *node) static void * chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, void *new_addr, size_t size, - size_t alignment, bool *zero) + size_t alignment, bool *zero, bool dalloc_node) { void *ret; extent_node_t *node; @@ -74,6 +74,7 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, bool zeroed; assert(new_addr == NULL || alignment == chunksize); + assert(dalloc_node || new_addr != NULL); alloc_size = size + alignment - chunksize; /* Beware size_t wrap-around. */ @@ -129,7 +130,8 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, } malloc_mutex_unlock(&arena->chunks_mtx); - if (node != NULL) + assert(!dalloc_node || node != NULL); + if (dalloc_node && node != NULL) arena_node_dalloc(arena, node); if (*zero) { if (!zeroed) @@ -153,8 +155,8 @@ chunk_alloc_core_dss(arena_t *arena, void *new_addr, size_t size, void *ret; if ((ret = chunk_recycle(arena, &arena->chunks_szad_dss, - &arena->chunks_ad_dss, false, new_addr, size, alignment, zero)) != - NULL) + &arena->chunks_ad_dss, false, new_addr, size, alignment, zero, + true)) != NULL) return (ret); ret = chunk_alloc_dss(arena, new_addr, size, alignment, zero); return (ret); @@ -177,11 +179,6 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment, assert(alignment != 0); assert((alignment & chunksize_mask) == 0); - /* cache. */ - if ((ret = chunk_recycle(arena, &arena->chunks_szad_cache, - &arena->chunks_ad_cache, true, new_addr, size, alignment, zero)) != - NULL) - return (ret); /* "primary" dss. */ if (have_dss && dss_prec == dss_prec_primary && (ret = chunk_alloc_core_dss(arena, new_addr, size, alignment, zero)) != @@ -190,7 +187,7 @@ chunk_alloc_core(arena_t *arena, void *new_addr, size_t size, size_t alignment, /* mmap. */ if (!config_munmap && (ret = chunk_recycle(arena, &arena->chunks_szad_mmap, &arena->chunks_ad_mmap, false, new_addr, - size, alignment, zero)) != NULL) + size, alignment, zero, true)) != NULL) return (ret); /* * Requesting an address is not implemented for chunk_alloc_mmap(), so @@ -231,19 +228,18 @@ chunk_alloc_base(size_t size) } void * -chunk_alloc_arena(chunk_alloc_t *chunk_alloc, chunk_dalloc_t *chunk_dalloc, - unsigned arena_ind, void *new_addr, size_t size, size_t alignment, - bool *zero) +chunk_alloc_cache(arena_t *arena, void *new_addr, size_t size, size_t alignment, + bool *zero, bool dalloc_node) { - void *ret; - ret = chunk_alloc(new_addr, size, alignment, zero, arena_ind); - if (ret == NULL) - return (NULL); - if (config_valgrind) - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + assert(size != 0); + assert((size & chunksize_mask) == 0); + assert(alignment != 0); + assert((alignment & chunksize_mask) == 0); - return (ret); + return (chunk_recycle(arena, &arena->chunks_szad_cache, + &arena->chunks_ad_cache, true, new_addr, size, alignment, zero, + dalloc_node)); } static arena_t * @@ -262,7 +258,27 @@ chunk_arena_get(unsigned arena_ind) return (arena); } -/* Default arena chunk allocation routine in the absence of user override. */ +static void * +chunk_alloc_arena(arena_t *arena, void *new_addr, size_t size, size_t alignment, + bool *zero) +{ + void *ret; + + ret = chunk_alloc_core(arena, new_addr, size, alignment, zero, + arena->dss_prec); + if (ret == NULL) + return (NULL); + if (config_valgrind) + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + + return (ret); +} + +/* + * Default arena chunk allocation routine in the absence of user override. This + * function isn't actually used by jemalloc, but it does the right thing if the + * application passes calls through to it during chunk allocation. + */ void * chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, unsigned arena_ind) @@ -270,8 +286,21 @@ chunk_alloc_default(void *new_addr, size_t size, size_t alignment, bool *zero, arena_t *arena; arena = chunk_arena_get(arena_ind); - return (chunk_alloc_core(arena, new_addr, size, alignment, zero, - arena->dss_prec)); + return (chunk_alloc_arena(arena, new_addr, size, alignment, zero)); +} + +void * +chunk_alloc_wrapper(arena_t *arena, chunk_alloc_t *chunk_alloc, void *new_addr, + size_t size, size_t alignment, bool *zero) +{ + void *ret; + + ret = chunk_alloc(new_addr, size, alignment, zero, arena->ind); + if (ret == NULL) + return (NULL); + if (config_valgrind && chunk_alloc != chunk_alloc_default) + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(chunk, chunksize); + return (ret); } void @@ -355,8 +384,8 @@ label_return: malloc_mutex_unlock(&arena->chunks_mtx); } -static void -chunk_cache(arena_t *arena, void *chunk, size_t size) +void +chunk_dalloc_cache(arena_t *arena, void *chunk, size_t size) { assert(chunk != NULL); @@ -366,19 +395,11 @@ chunk_cache(arena_t *arena, void *chunk, size_t size) chunk_record(arena, &arena->chunks_szad_cache, &arena->chunks_ad_cache, true, chunk, size, false); -} - -/* Default arena chunk deallocation routine in the absence of user override. */ -bool -chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) -{ - - chunk_cache(chunk_arena_get(arena_ind), chunk, size); - return (false); + arena_maybe_purge(arena); } void -chunk_unmap(arena_t *arena, void *chunk, size_t size, bool zeroed) +chunk_dalloc_arena(arena_t *arena, void *chunk, size_t size, bool zeroed) { assert(chunk != NULL); @@ -395,6 +416,29 @@ chunk_unmap(arena_t *arena, void *chunk, size_t size, bool zeroed) } } +/* + * Default arena chunk deallocation routine in the absence of user override. + * This function isn't actually used by jemalloc, but it does the right thing if + * the application passes calls through to it during chunk deallocation. + */ +bool +chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind) +{ + + chunk_dalloc_arena(chunk_arena_get(arena_ind), chunk, size, false); + return (false); +} + +void +chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc, void *chunk, + size_t size) +{ + + chunk_dalloc(chunk, size, arena->ind); + if (config_valgrind && chunk_dalloc != chunk_dalloc_default) + JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); +} + static rtree_node_elm_t * chunks_rtree_node_alloc(size_t nelms) { From 35e3fd9a63a9d24276eab24bf84edb3d9e856732 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 18 Feb 2015 16:51:51 -0800 Subject: [PATCH 241/352] Fix a compilation error and an incorrect assertion. --- src/chunk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/chunk.c b/src/chunk.c index 08f21f6a..9474a158 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -130,7 +130,7 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, } malloc_mutex_unlock(&arena->chunks_mtx); - assert(!dalloc_node || node != NULL); + assert(dalloc_node || node != NULL); if (dalloc_node && node != NULL) arena_node_dalloc(arena, node); if (*zero) { @@ -299,7 +299,7 @@ chunk_alloc_wrapper(arena_t *arena, chunk_alloc_t *chunk_alloc, void *new_addr, if (ret == NULL) return (NULL); if (config_valgrind && chunk_alloc != chunk_alloc_default) - JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(chunk, chunksize); + JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, chunksize); return (ret); } From 970fcfbca5dffda921394172c7298d274eebfd0e Mon Sep 17 00:00:00 2001 From: Dave Huseby Date: Mon, 9 Feb 2015 21:46:54 -0800 Subject: [PATCH 242/352] adding support for bitrig --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 7a694a20..4ac7ac82 100644 --- a/configure.ac +++ b/configure.ac @@ -283,7 +283,7 @@ case "${host}" in abi="elf" AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) ;; - *-*-openbsd*) + *-*-openbsd*|*-*-bitrig*) CFLAGS="$CFLAGS" abi="elf" AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) From 7c46fd59cce6afb14cdc6c819f662b6e81638f84 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 4 Mar 2015 21:48:01 +0900 Subject: [PATCH 243/352] Make --without-export actually work 9906660 added a --without-export configure option to avoid exporting jemalloc symbols, but the option didn't actually work. --- .../jemalloc/internal/jemalloc_internal_defs.h.in | 6 ++++++ include/jemalloc/jemalloc_macros.h.in | 14 +++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 0f0db8a1..191abc52 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -232,4 +232,10 @@ /* Adaptive mutex support in pthreads. */ #undef JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP +/* + * If defined, jemalloc symbols are not exported (doesn't work when + * JEMALLOC_PREFIX is not defined). + */ +#undef JEMALLOC_EXPORT + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in index 7d1dcf4a..72f2a082 100644 --- a/include/jemalloc/jemalloc_macros.h.in +++ b/include/jemalloc/jemalloc_macros.h.in @@ -32,16 +32,20 @@ #ifdef JEMALLOC_HAVE_ATTR # define JEMALLOC_ATTR(s) __attribute__((s)) -# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# ifndef JEMALLOC_EXPORT +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# endif # define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) # define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) # define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) #elif _MSC_VER # define JEMALLOC_ATTR(s) -# ifdef DLLEXPORT -# define JEMALLOC_EXPORT __declspec(dllexport) -# else -# define JEMALLOC_EXPORT __declspec(dllimport) +# ifndef JEMALLOC_EXPORT +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif # endif # define JEMALLOC_ALIGNED(s) __declspec(align(s)) # define JEMALLOC_SECTION(s) __declspec(allocate(s)) From 4d871f73af6b8310564dfcb63357dbfe8b1a1529 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Wed, 4 Mar 2015 10:54:10 +0900 Subject: [PATCH 244/352] Preserve LastError when calling TlsGetValue TlsGetValue has a semantic difference with pthread_getspecific, in that it can return a non-error NULL value, so it always sets the LastError. But allocator callers may not be expecting calling e.g. free() to change the value of the last error, so preserve it. --- include/jemalloc/internal/tsd.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/jemalloc/internal/tsd.h b/include/jemalloc/internal/tsd.h index dbb91a2e..62a887e6 100644 --- a/include/jemalloc/internal/tsd.h +++ b/include/jemalloc/internal/tsd.h @@ -277,9 +277,11 @@ a_name##tsd_set(a_type *val) \ a_attr bool \ a_name##tsd_cleanup_wrapper(void) \ { \ - a_name##tsd_wrapper_t *wrapper; \ + DWORD error = GetLastError(); \ + a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ + TlsGetValue(a_name##tsd_tsd); \ + SetLastError(error); \ \ - wrapper = (a_name##tsd_wrapper_t *)TlsGetValue(a_name##tsd_tsd);\ if (wrapper == NULL) \ return (false); \ if (a_cleanup != malloc_tsd_no_cleanup && \ @@ -307,8 +309,10 @@ a_name##tsd_wrapper_set(a_name##tsd_wrapper_t *wrapper) \ a_attr a_name##tsd_wrapper_t * \ a_name##tsd_wrapper_get(void) \ { \ + DWORD error = GetLastError(); \ a_name##tsd_wrapper_t *wrapper = (a_name##tsd_wrapper_t *) \ TlsGetValue(a_name##tsd_tsd); \ + SetLastError(error); \ \ if (unlikely(wrapper == NULL)) { \ wrapper = (a_name##tsd_wrapper_t *) \ From f044bb219e9bfcc585f64f097e5ab0b5837c0451 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 6 Mar 2015 20:05:16 -0800 Subject: [PATCH 245/352] Change default chunk size from 4 MiB to 256 KiB. Recent changes have improved huge allocation scalability, which removes upward pressure to set the chunk size so large that huge allocations are rare. Smaller chunks are more likely to completely drain, so set the default to the smallest size that doesn't leave excessive unusable trailing space in chunk headers. --- doc/jemalloc.xml.in | 26 +++++++++++++------------- include/jemalloc/internal/chunk.h | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index b392fa9e..747e03f4 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -571,8 +571,8 @@ for (i = 0; i < nbins; i++) { both large or both huge. In such cases shrinkage always succeeds, but growth only succeeds if the trailing memory is currently available. - Assuming 4 MiB chunks, 4 KiB pages, and a 16-byte quantum on a 64-bit - system, the size classes in each category are as shown in Assuming 256 KiB chunks, 4 KiB pages, and a 16-byte quantum on a + 64-bit system, the size classes in each category are as shown in . @@ -627,7 +627,7 @@ for (i = 0; i < nbins; i++) { [10 KiB, 12 KiB, 14 KiB] - Large + Large 2 KiB [16 KiB] @@ -645,7 +645,12 @@ for (i = 0; i < nbins; i++) { 32 KiB - [160 KiB, 192 KiB, 224 KiB, 256 KiB] + [160 KiB, 192 KiB, 224 KiB] + + + Huge + 32 KiB + [256 KiB] 64 KiB @@ -653,20 +658,15 @@ for (i = 0; i < nbins; i++) { 128 KiB - [640 KiB, 768 KiB, 896 KiB, 1024 KiB] + [640 KiB, 768 KiB, 896 KiB, 1 MiB] 256 KiB - [1280 KiB, 1536 KiB, 1792 KiB, 2048 KiB] + [1280 KiB, 1536 KiB, 1792 KiB, 2 MiB] 512 KiB - [2560 KiB, 3072 KiB, 3584 KiB] - - - Huge - 512 KiB - [4 MiB] + [2560 KiB, 3 MiB, 3584 KiB, 4 MiB] 1 MiB @@ -907,7 +907,7 @@ for (i = 0; i < nbins; i++) { Virtual memory chunk size (log base 2). If a chunk size outside the supported size range is specified, the size is silently clipped to the minimum/maximum supported size. The default - chunk size is 4 MiB (2^22). + chunk size is 256 KiB (2^18). diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 1a968a58..1af5b24b 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -5,7 +5,7 @@ * Size and alignment of memory chunks that are allocated by the OS's virtual * memory system. */ -#define LG_CHUNK_DEFAULT 22 +#define LG_CHUNK_DEFAULT 18 /* Return the chunk address for allocation address a. */ #define CHUNK_ADDR2BASE(a) \ From 5707d6f952c71baa2f19102479859012982ac821 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 6 Mar 2015 17:14:05 -0800 Subject: [PATCH 246/352] Quantize szad trees by size class. Treat sizes that round down to the same size class as size-equivalent in trees that are used to search for first best fit, so that there are only as many "firsts" as there are size classes. This comes closer to the ideal of first fit. --- src/arena.c | 36 +++++++++++++++++++++++++++--------- src/base.c | 5 +++-- src/chunk.c | 2 +- src/extent.c | 8 +++++++- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/src/arena.c b/src/arena.c index 78aa1ae0..34329a62 100644 --- a/src/arena.c +++ b/src/arena.c @@ -59,21 +59,35 @@ JEMALLOC_INLINE_C int arena_avail_comp(arena_chunk_map_misc_t *a, arena_chunk_map_misc_t *b) { int ret; + uintptr_t a_miscelm = (uintptr_t)a; size_t a_size; size_t b_size = arena_miscelm_to_bits(b) & ~PAGE_MASK; - uintptr_t a_miscelm = (uintptr_t)a; - uintptr_t b_miscelm = (uintptr_t)b; + index_t a_index, b_index; - if (a_miscelm & CHUNK_MAP_KEY) + if (a_miscelm & CHUNK_MAP_KEY) { a_size = a_miscelm & ~PAGE_MASK; - else + assert(a_size == s2u(a_size)); + } else a_size = arena_miscelm_to_bits(a) & ~PAGE_MASK; - ret = (a_size > b_size) - (a_size < b_size); + /* + * Compute the index of the largest size class that the run can satisfy + * a request for. + */ + a_index = size2index(a_size + 1) - 1; + b_index = size2index(b_size + 1) - 1; + + /* + * Compare based on size class index rather than size, in order to + * sort equally useful runs only by address. + */ + ret = (a_index > b_index) - (a_index < b_index); if (ret == 0) { - if (!(a_miscelm & CHUNK_MAP_KEY)) + if (!(a_miscelm & CHUNK_MAP_KEY)) { + uintptr_t b_miscelm = (uintptr_t)b; + ret = (a_miscelm > b_miscelm) - (a_miscelm < b_miscelm); - else { + } else { /* * Treat keys as if they are lower than anything else. */ @@ -898,8 +912,10 @@ arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { arena_chunk_map_misc_t *miscelm; arena_chunk_map_misc_t *key; + size_t usize; - key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); + usize = s2u(size); + key = (arena_chunk_map_misc_t *)(usize | CHUNK_MAP_KEY); miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (miscelm != NULL) { arena_run_t *run = &miscelm->run; @@ -949,7 +965,8 @@ arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind) arena_chunk_map_misc_t *miscelm; arena_chunk_map_misc_t *key; - key = (arena_chunk_map_misc_t *)(size | CHUNK_MAP_KEY); + assert(size == s2u(size)); + key = (arena_chunk_map_misc_t *)(PAGE_CEILING(size) | CHUNK_MAP_KEY); miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); if (miscelm != NULL) { run = &miscelm->run; @@ -2778,6 +2795,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info) bin_info->reg_interval; } assert(actual_nregs > 0); + assert(actual_run_size == s2u(actual_run_size)); /* Copy final settings. */ bin_info->run_size = actual_run_size; diff --git a/src/base.c b/src/base.c index 33e8b6f5..01c62df4 100644 --- a/src/base.c +++ b/src/base.c @@ -73,7 +73,7 @@ void * base_alloc(size_t size) { void *ret; - size_t csize; + size_t csize, usize; extent_node_t *node; extent_node_t key; @@ -83,7 +83,8 @@ base_alloc(size_t size) */ csize = CACHELINE_CEILING(size); - extent_node_init(&key, NULL, NULL, csize, false); + usize = s2u(csize); + extent_node_init(&key, NULL, NULL, usize, false); malloc_mutex_lock(&base_mtx); node = extent_tree_szad_nsearch(&base_avail_szad, &key); if (node != NULL) { diff --git a/src/chunk.c b/src/chunk.c index 9474a158..972fecde 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -76,7 +76,7 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, assert(new_addr == NULL || alignment == chunksize); assert(dalloc_node || new_addr != NULL); - alloc_size = size + alignment - chunksize; + alloc_size = CHUNK_CEILING(s2u(size + alignment - chunksize)); /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); diff --git a/src/extent.c b/src/extent.c index f98e77e2..e16f8f6c 100644 --- a/src/extent.c +++ b/src/extent.c @@ -9,8 +9,14 @@ extent_szad_comp(extent_node_t *a, extent_node_t *b) int ret; size_t a_size = extent_node_size_get(a); size_t b_size = extent_node_size_get(b); + /* + * Compute the index of the largest size class that the chunk can + * satisfy a request for. + */ + size_t a_index = size2index(a_size + 1) - 1; + size_t b_index = size2index(b_size + 1) - 1; - ret = (a_size > b_size) - (a_size < b_size); + ret = (a_index > b_index) - (a_index < b_index); if (ret == 0) { uintptr_t a_addr = (uintptr_t)extent_node_addr_get(a); uintptr_t b_addr = (uintptr_t)extent_node_addr_get(b); From 97c04a93838c4001688fe31bf018972b4696efe2 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 6 Mar 2015 19:57:36 -0800 Subject: [PATCH 247/352] Use first-fit rather than first-best-fit run/chunk allocation. This tends to more effectively pack active memory toward low addresses. However, additional tree searches are required in many cases, so whether this change stands the test of time will depend on real-world benchmarks. --- include/jemalloc/internal/arena.h | 2 +- src/arena.c | 76 ++++++++++++++++++++----------- src/chunk.c | 43 +++++++++++++++-- 3 files changed, 89 insertions(+), 32 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 42086ca1..50b296e4 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -312,7 +312,7 @@ struct arena_s { /* * Size/address-ordered tree of this arena's available runs. The tree - * is used for first-best-fit run allocation. + * is used for first-fit run allocation. */ arena_avail_tree_t runs_avail; diff --git a/src/arena.c b/src/arena.c index 34329a62..6f4197b2 100644 --- a/src/arena.c +++ b/src/arena.c @@ -907,23 +907,55 @@ arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, return (err); } +/* Do first-fit run selection. */ +static arena_run_t * +arena_run_first_fit(arena_t *arena, size_t size) +{ + arena_run_t *run; + index_t index, max_index; + + assert(size == s2u(size)); + assert(size == PAGE_CEILING(size)); + + /* + * Iterate over all size classes that are at least large enough to + * satisfy the request, search for the lowest run of each size class, + * and choose the lowest of the runs found. + */ + run = NULL; + for (index = size2index(size), max_index = size2index(arena_maxclass); + index <= max_index;) { + arena_run_t *currun; + arena_chunk_t *currun_chunk; + size_t currun_pageind, currun_size; + size_t usize = PAGE_CEILING(index2size(index)); + arena_chunk_map_misc_t *key = (arena_chunk_map_misc_t *)(usize | + CHUNK_MAP_KEY); + arena_chunk_map_misc_t *miscelm = + arena_avail_tree_nsearch(&arena->runs_avail, key); + if (miscelm == NULL) + break; + currun = &miscelm->run; + if (run == NULL || (uintptr_t)currun < (uintptr_t)run) + run = currun; + currun_chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(currun); + currun_pageind = arena_miscelm_to_pageind(miscelm); + currun_size = arena_mapbits_unallocated_size_get(currun_chunk, + currun_pageind); + assert(size2index(currun_size) + 1 > index); + index = size2index(currun_size) + 1; + } + + return (run); +} + static arena_run_t * arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { - arena_chunk_map_misc_t *miscelm; - arena_chunk_map_misc_t *key; - size_t usize; - - usize = s2u(size); - key = (arena_chunk_map_misc_t *)(usize | CHUNK_MAP_KEY); - miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); - if (miscelm != NULL) { - arena_run_t *run = &miscelm->run; - arena_run_split_large(arena, &miscelm->run, size, zero); - return (run); - } - - return (NULL); + arena_run_t *run = arena_run_first_fit(arena, s2u(size)); + if (run != NULL) + arena_run_split_large(arena, run, size, zero); + return (run); } static arena_run_t * @@ -961,20 +993,10 @@ arena_run_alloc_large(arena_t *arena, size_t size, bool zero) static arena_run_t * arena_run_alloc_small_helper(arena_t *arena, size_t size, index_t binind) { - arena_run_t *run; - arena_chunk_map_misc_t *miscelm; - arena_chunk_map_misc_t *key; - - assert(size == s2u(size)); - key = (arena_chunk_map_misc_t *)(PAGE_CEILING(size) | CHUNK_MAP_KEY); - miscelm = arena_avail_tree_nsearch(&arena->runs_avail, key); - if (miscelm != NULL) { - run = &miscelm->run; + arena_run_t *run = arena_run_first_fit(arena, PAGE_CEILING(size)); + if (run != NULL) arena_run_split_small(arena, run, size, binind); - return (run); - } - - return (NULL); + return (run); } static arena_run_t * diff --git a/src/chunk.c b/src/chunk.c index 972fecde..875fa4cc 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -62,6 +62,39 @@ chunk_deregister(const void *chunk, const extent_node_t *node) } } +/* Do first-fit chunk selection. */ +static extent_node_t * +chunk_first_fit(arena_t *arena, extent_tree_t *chunks_szad, size_t size) +{ + extent_node_t *node; + index_t index; + + assert(size == CHUNK_CEILING(size)); + + /* + * Iterate over all size classes that are at least large enough to + * satisfy the request, search for the lowest chunk of each size class, + * and choose the lowest of the chunks found. + */ + node = NULL; + for (index = size2index(size); index < NSIZES;) { + extent_node_t *curnode; + extent_node_t key; + extent_node_init(&key, arena, NULL, + CHUNK_CEILING(index2size(index)), false); + curnode = extent_tree_szad_nsearch(chunks_szad, &key); + if (curnode == NULL) + break; + if (node == NULL || (uintptr_t)extent_node_addr_get(curnode) < + (uintptr_t)extent_node_addr_get(node)) + node = curnode; + assert(size2index(extent_node_size_get(curnode)) + 1 > index); + index = size2index(extent_node_size_get(curnode)) + 1; + } + + return (node); +} + static void * chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, bool cache, void *new_addr, size_t size, @@ -69,7 +102,6 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, { void *ret; extent_node_t *node; - extent_node_t key; size_t alloc_size, leadsize, trailsize; bool zeroed; @@ -80,10 +112,13 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, /* Beware size_t wrap-around. */ if (alloc_size < size) return (NULL); - extent_node_init(&key, arena, new_addr, alloc_size, false); malloc_mutex_lock(&arena->chunks_mtx); - node = (new_addr != NULL) ? extent_tree_ad_search(chunks_ad, &key) : - extent_tree_szad_nsearch(chunks_szad, &key); + if (new_addr != NULL || size == chunksize) { + extent_node_t key; + extent_node_init(&key, arena, new_addr, alloc_size, false); + node = extent_tree_ad_search(chunks_ad, &key); + } else + node = chunk_first_fit(arena, chunks_szad, alloc_size); if (node == NULL || (new_addr != NULL && extent_node_size_get(node) < size)) { malloc_mutex_unlock(&arena->chunks_mtx); From 04ca7580dbc409915de05cb1cee12a369e898590 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 6 Mar 2015 23:25:13 -0800 Subject: [PATCH 248/352] Fix a chunk_recycle() regression. This regression was introduced by 97c04a93838c4001688fe31bf018972b4696efe2 (Use first-fit rather than first-best-fit run/chunk allocation.). --- src/chunk.c | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/chunk.c b/src/chunk.c index 875fa4cc..fb8cd413 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -64,13 +64,22 @@ chunk_deregister(const void *chunk, const extent_node_t *node) /* Do first-fit chunk selection. */ static extent_node_t * -chunk_first_fit(arena_t *arena, extent_tree_t *chunks_szad, size_t size) +chunk_first_fit(arena_t *arena, extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, size_t size) { extent_node_t *node; index_t index; assert(size == CHUNK_CEILING(size)); + if (size == chunksize) { + /* + * Any chunk will suffice, so simply select the one lowest in + * memory. + */ + return (extent_tree_ad_first(chunks_ad)); + } + /* * Iterate over all size classes that are at least large enough to * satisfy the request, search for the lowest chunk of each size class, @@ -113,12 +122,14 @@ chunk_recycle(arena_t *arena, extent_tree_t *chunks_szad, if (alloc_size < size) return (NULL); malloc_mutex_lock(&arena->chunks_mtx); - if (new_addr != NULL || size == chunksize) { + if (new_addr != NULL) { extent_node_t key; extent_node_init(&key, arena, new_addr, alloc_size, false); node = extent_tree_ad_search(chunks_ad, &key); - } else - node = chunk_first_fit(arena, chunks_szad, alloc_size); + } else { + node = chunk_first_fit(arena, chunks_szad, chunks_ad, + alloc_size); + } if (node == NULL || (new_addr != NULL && extent_node_size_get(node) < size)) { malloc_mutex_unlock(&arena->chunks_mtx); From 54673fd8d719e081536fb531417cd9060de895f0 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 23 Feb 2015 22:28:43 -0800 Subject: [PATCH 249/352] Update ChangeLog. --- ChangeLog | 151 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 150 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index d56ee999..ef7dbfdb 100644 --- a/ChangeLog +++ b/ChangeLog @@ -5,6 +5,155 @@ found in the git revision history: https://github.com/jemalloc/jemalloc +* 4.0.0 (XXX) See https://github.com/jemalloc/jemalloc/milestones/4.0.0 for + remaining work. + + This version contains many speed and space optimizations, both minor and + major. The major themes are generalization, unification, and simplification. + Although many of these optimizations cause no visible behavior change, their + cumulative effect is substantial. + + New features: + - Normalize size class spacing to be consistent across the complete size + range. By default there are four size classes per size doubling, but this + is now configurable via the --with-lg-size-class-group option. Also add the + --with-lg-page, --with-lg-page-sizes, --with-lg-quantum, and + --with-lg-tiny-min options, which can be used to tweak page and size class + settings. Impacts: + + Worst case performance for incrementally growing/shrinking reallocation + is improved because there are far fewer size classes, and therefore + copying happens less often. + + Internal fragmentation is limited to 20% for all but the smallest size + classes (those less than four times the quantum). (1B + 4 KiB) + and (1B + 4 MiB) previously suffered nearly 50% internal fragmentation. + + Chunk fragmentation tends to be lower because there are fewer distinct run + sizes to pack. + - Add support for explicit tcaches. The "tcache.create", "tcache.flush", and + "tcache.destroy" mallctls control tcache lifetime and flushing, and the + MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to the *allocx() API + control which tcache is used for each operation. + - Implement per thread heap profiling, as well as the ability to + enable/disable heap profiling on a per thread basis. Add the "prof.reset", + "prof.lg_sample", "thread.prof.name", "thread.prof.active", + "opt.prof_thread_active_init", "prof.thread_active_init", and + "thread.prof.active" mallctls. + - Add support for per arena application-specified chunk allocators, configured + via the "arena.chunk.alloc" and "arena.chunk.dalloc" mallctls. + - Refactor huge allocation to be managed by arenas, so that arenas now + function as general purpose independent allocators. This is important in + the context of user-specified chunk allocators, aside from the scalability + benefits. Related new statistics: + + The "stats.arenas..huge.allocated", "stats.arenas..huge.nmalloc", + "stats.arenas..huge.ndalloc", and "stats.arenas..huge.nrequests" + mallctls provide high level per arena huge allocation statistics. + + The "arenas.nhchunks", "arenas.hchunks..size", + "stats.arenas..hchunks..nmalloc", + "stats.arenas..hchunks..ndalloc", + "stats.arenas..hchunks..nrequests", and + "stats.arenas..hchunks..curhchunks" mallctls provide per size class + statistics. + - Add the 'util' column to malloc_stats_print() output, which reports the + proportion of available regions that are currently in use for each small + size class. + - Add "alloc" and "free" modes for for junk filling (see the "opt.junk" + mallctl), so that it is possible to separately enable junk filling for + allocation versus deallocation. + - Add the jemalloc-config script, which provides information about how + jemalloc was configured, and how to integrate it into application builds. + - Add metadata statistics, which are accessible via the "stats.metadata", + "stats.arenas..metadata.mapped", and + "stats.arenas..metadata.allocated" mallctls. + - Add the "prof.gdump" mallctl, which makes it possible to toggle the gdump + feature on/off during program execution. + - Add sdallocx(), which implements sized deallocation. The primary + optimization over dallocx() is the removal of a metadata read, which often + suffers an L1 cache miss. + - Add missing header includes in jemalloc/jemalloc.h, so that applications + only have to #include . + - Add support for additional platforms: + + Bitrig + + Cygwin + + DragonFlyBSD + + iOS + + OpenBSD + + OpenRISC/or1k + + Optimizations: + - Switch run and chunk allocation from first-best-fit (among best-fit + candidates, choose the lowest in memory) to first-fit (among all candidates, + choose the lowest in memory). This tends to reduce chunk and virtual memory + fragmentation, respectively. + - Maintain dirty runs in per arena LRUs rather than in per arena trees of + dirty-run-containing chunks. In practice this change significantly reduces + dirty page purging volume. + - Integrate whole chunks into the unused dirty page purging machinery. This + reduces the cost of repeated huge allocation/deallocation, because it + effectively introduces a cache of chunks. + - Split the arena chunk map into two separate arrays, in order to increase + cache locality for the frequently accessed bits. + - Move small run metadata out of runs, into arena chunk headers. This reduces + run fragmentation, smaller runs reduce external fragmentation for small size + classes, and packed (less uniformly aligned) metadata layout improves CPU + cache set distribution. + - Micro-optimize the fast paths for the public API functions. + - Refactor thread-specific data to reside in a single structure. This assures + that only a single TLS read is necessary per call into the public API. + - Implement in-place huge allocation growing and shrinking. + - Refactor rtree (radix tree for chunk lookups) to be lock-free, and make + additional optimizations that reduce maximum lookup depth to one or two + levels. This resolves what was a concurrency bottleneck for per arena huge + allocation, because a global data structure is critical for determining + which arenas own which huge allocations. + + Incompatible changes: + - Replace --enable-cc-silence with --disable-cc-silence to suppress spurious + warnings by default. + - Assure that the constness of malloc_usable_size()'s return type matches that + of the system implementation. + - Change the heap profile dump format to support per thread heap profiling, + and enhance pprof with the --thread= option. As a result, the bundled + pprof must now be used rather than the upstream (gperftools) pprof. + - Disable "opt.prof_final" by default, in order to avoid atexit(3), which can + internally deadlock on some platforms. + - Change the "arenas.nlruns" mallctl type from size_t to unsigned. + - Replace the "stats.arenas..bins..allocated" mallctl with + "stats.arenas..bins..curregs". + - Ignore MALLOC_CONF in set{uid,gid,cap} binaries. + - Ignore MALLOCX_ARENA(a) in dallocx(), in favor of using the + MALLOCX_TCACHE(tc) and MALLOCX_TCACHE_NONE flags to control tcache usage. + + Removed features: + - Remove the *allocm() API, which is superseded by the *allocx() API. + - Remove the --enable-dss options, and make dss non-optional on all platforms + which support sbrk(2). + - Remove the "arenas.purge" mallctl, which was obsoleted by the + "arena..purge" mallctl in 3.1.0. + - Remove the unnecessary "opt.valgrind" mallctl; jemalloc automatically + detects whether it is running inside Valgrind. + - Remove the "stats.huge.allocated", "stats.huge.nmalloc", and + "stats.huge.ndalloc" mallctls. + - Remove the --enable-mremap option. + - Remove the --enable-ivsalloc option, and merge its functionality into + --enable-debug. + - Remove the "stats.chunks.current", "stats.chunks.total", and + "stats.chunks.high" mallctls. + + Bug fixes: + - Fix the cactive statistic to decrease (rather than increase) when active + memory decreases. This regression was first released in 3.5.0. + - Fix OOM handling in memalign() and valloc(). A variant of this bug existed + in all releases since 2.0.0, which introduced these functions. + - Fix the "arena..dss" mallctl to return an error if "primary" or + "secondary" precedence is specified, but sbrk(2) is not supported. + - Fix fallback lg_floor() implementations to handle extremely large inputs. + - Ensure the default purgeable zone is after the default zone on OS X. + - Fix latent bugs in atomic_*(). + - Fix the "arena..dss" mallctl to handle read-only calls. + - Fix tls_model configuration to enable the initial-exec model when possible. + - Mark malloc_conf as a weak symbol so that the application can override it. + - Correctly detect glibc's adaptive pthread mutexes. + - Fix the --without-export configure option. + * 3.6.0 (March 31, 2014) This version contains a critical bug fix for a regression present in 3.5.0 and @@ -21,7 +170,7 @@ found in the git revision history: backtracing to be reliable. - Use dss allocation precedence for huge allocations as well as small/large allocations. - - Fix test assertion failure message formatting. This bug did not manifect on + - Fix test assertion failure message formatting. This bug did not manifest on x86_64 systems because of implementation subtleties in va_list. - Fix inconsequential test failures for hash and SFMT code. From 38e42d311c1844a66e8ced84551621de41e42b85 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 10 Mar 2015 18:15:40 -0700 Subject: [PATCH 250/352] Refactor dirty run linkage to reduce sizeof(extent_node_t). --- include/jemalloc/internal/arena.h | 50 ++++++++--- include/jemalloc/internal/extent.h | 12 +-- include/jemalloc/internal/private_symbols.txt | 1 + src/arena.c | 89 ++++++++++--------- 4 files changed, 95 insertions(+), 57 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 50b296e4..de298e55 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -23,6 +23,7 @@ */ #define LG_DIRTY_MULT_DEFAULT 3 +typedef struct arena_runs_dirty_link_s arena_runs_dirty_link_t; typedef struct arena_run_s arena_run_t; typedef struct arena_chunk_map_bits_s arena_chunk_map_bits_t; typedef struct arena_chunk_map_misc_s arena_chunk_map_misc_t; @@ -120,6 +121,10 @@ struct arena_chunk_map_bits_s { #define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED }; +struct arena_runs_dirty_link_s { + qr(arena_runs_dirty_link_t) rd_link; +}; + /* * Each arena_chunk_map_misc_t corresponds to one page within the chunk, just * like arena_chunk_map_bits_t. Two separate arrays are stored within each @@ -131,13 +136,13 @@ struct arena_chunk_map_misc_s { * * 1) arena_t's runs_avail tree. * 2) arena_run_t conceptually uses this linkage for in-use non-full - * runs, rather than directly embedding linkage. + * runs, rather than directly embedding linkage. */ rb_node(arena_chunk_map_misc_t) rb_link; union { /* Linkage for list of dirty runs. */ - qr(arena_chunk_map_misc_t) rd_link; + arena_runs_dirty_link_t rd; /* Profile counters, used for large object runs. */ prof_tctx_t *prof_tctx; @@ -324,15 +329,27 @@ struct arena_s { * * LRU-----------------------------------------------------------MRU * - * ______________ ___ ___ - * ...-->|chunks_cache|<--------->|c|<-------------------->|c|<--... - * -------------- |h| |h| - * ____________ _____ |u| _____ _____ |u| - * ...-->|runs_dirty|<-->|run|<-->|n|<-->|run|<-->|run|<-->|n|<--... - * ------------ ----- |k| ----- ----- |k| - * --- --- + * /------------------\ + * | arena | + * | | + * | /------------\ | /-----------\ + * ...---->|chunks_cache|<---------------------->| chunk |<--... + * | \------------/ | | | + * | | | | + * | | /---\ /---\ | | + * | | |run| |run| | | + * | | | | | | | | + * | /----------\ | |---| |---| | /-----\ | + * ...----->|runs_dirty|<---->|rd |<---->|rd |<---->|rdelm|<-----... + * | \----------/ | |---| |---| | \-----/ | + * | | | | | | | | + * | | | | | | | | + * | | \---/ \---/ | | + * | | | | + * | | | | + * \------------------/ \-----------/ */ - arena_chunk_map_misc_t runs_dirty; + arena_runs_dirty_link_t runs_dirty; extent_node_t chunks_cache; /* Extant huge allocations. */ @@ -465,6 +482,7 @@ arena_chunk_map_misc_t *arena_miscelm_get(arena_chunk_t *chunk, size_t pageind); size_t arena_miscelm_to_pageind(arena_chunk_map_misc_t *miscelm); void *arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm); +arena_chunk_map_misc_t *arena_rd_to_miscelm(arena_runs_dirty_link_t *rd); arena_chunk_map_misc_t *arena_run_to_miscelm(arena_run_t *run); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbitsp_read(size_t *mapbitsp); @@ -556,6 +574,18 @@ arena_miscelm_to_rpages(arena_chunk_map_misc_t *miscelm) return ((void *)((uintptr_t)chunk + (pageind << LG_PAGE))); } +JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * +arena_rd_to_miscelm(arena_runs_dirty_link_t *rd) +{ + arena_chunk_map_misc_t *miscelm = (arena_chunk_map_misc_t + *)((uintptr_t)rd - offsetof(arena_chunk_map_misc_t, rd)); + + assert(arena_miscelm_to_pageind(miscelm) >= map_bias); + assert(arena_miscelm_to_pageind(miscelm) < chunk_npages); + + return (miscelm); +} + JEMALLOC_ALWAYS_INLINE arena_chunk_map_misc_t * arena_run_to_miscelm(arena_run_t *run) { diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 81ff40b8..5dbc04a5 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -34,7 +34,7 @@ struct extent_node_s { prof_tctx_t *en_prof_tctx; /* Linkage for arena's runs_dirty and chunks_cache rings. */ - arena_chunk_map_misc_t runs_dirty; + arena_runs_dirty_link_t rdelm; qr(extent_node_t) cc_link; union { @@ -79,7 +79,7 @@ void extent_node_init(extent_node_t *node, arena_t *arena, void *addr, size_t size, bool zeroed); void extent_node_dirty_linkage_init(extent_node_t *node); void extent_node_dirty_insert(extent_node_t *node, - arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty); + arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty); void extent_node_dirty_remove(extent_node_t *node); #endif @@ -186,16 +186,16 @@ JEMALLOC_INLINE void extent_node_dirty_linkage_init(extent_node_t *node) { - qr_new(&node->runs_dirty, rd_link); + qr_new(&node->rdelm, rd_link); qr_new(node, cc_link); } JEMALLOC_INLINE void extent_node_dirty_insert(extent_node_t *node, - arena_chunk_map_misc_t *runs_dirty, extent_node_t *chunks_dirty) + arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty) { - qr_meld(runs_dirty, &node->runs_dirty, rd_link); + qr_meld(runs_dirty, &node->rdelm, rd_link); qr_meld(chunks_dirty, node, cc_link); } @@ -203,7 +203,7 @@ JEMALLOC_INLINE void extent_node_dirty_remove(extent_node_t *node) { - qr_remove(&node->runs_dirty, rd_link); + qr_remove(&node->rdelm, rd_link); qr_remove(node, cc_link); } diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index 7c217c74..d086db18 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -81,6 +81,7 @@ arena_quarantine_junk_small arena_ralloc arena_ralloc_junk_large arena_ralloc_no_move +arena_rd_to_miscelm arena_redzone_corruption arena_run_regind arena_run_to_miscelm diff --git a/src/arena.c b/src/arena.c index 6f4197b2..5d792f99 100644 --- a/src/arena.c +++ b/src/arena.c @@ -136,8 +136,8 @@ arena_run_dirty_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - qr_new(miscelm, rd_link); - qr_meld(&arena->runs_dirty, miscelm, rd_link); + qr_new(&miscelm->rd, rd_link); + qr_meld(&arena->runs_dirty, &miscelm->rd, rd_link); arena->ndirty += npages; } @@ -153,7 +153,7 @@ arena_run_dirty_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, assert(arena_mapbits_dirty_get(chunk, pageind+npages-1) == CHUNK_MAP_DIRTY); - qr_remove(miscelm, rd_link); + qr_remove(&miscelm->rd, rd_link); assert(arena->ndirty >= npages); arena->ndirty -= npages; } @@ -1056,22 +1056,23 @@ static size_t arena_dirty_count(arena_t *arena) { size_t ndirty = 0; - arena_chunk_map_misc_t *runselm; + arena_runs_dirty_link_t *rdelm; extent_node_t *chunkselm; - for (runselm = qr_next(&arena->runs_dirty, rd_link), + for (rdelm = qr_next(&arena->runs_dirty, rd_link), chunkselm = qr_next(&arena->chunks_cache, cc_link); - runselm != &arena->runs_dirty; runselm = qr_next(runselm, - rd_link)) { + rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { size_t npages; - if (runselm == &chunkselm->runs_dirty) { + if (rdelm == &chunkselm->rdelm) { npages = extent_node_size_get(chunkselm) >> LG_PAGE; chunkselm = qr_next(chunkselm, cc_link); } else { - arena_chunk_t *chunk = (arena_chunk_t - *)CHUNK_ADDR2BASE(runselm); - size_t pageind = arena_miscelm_to_pageind(runselm); + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( + rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); assert(arena_mapbits_allocated_get(chunk, pageind) == 0); assert(arena_mapbits_large_get(chunk, pageind) == 0); @@ -1107,21 +1108,21 @@ arena_compute_npurge(arena_t *arena, bool all) static size_t arena_stash_dirty(arena_t *arena, bool all, size_t npurge, - arena_chunk_map_misc_t *purge_runs_sentinel, + arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { - arena_chunk_map_misc_t *runselm, *runselm_next; + arena_runs_dirty_link_t *rdelm, *rdelm_next; extent_node_t *chunkselm; size_t nstashed = 0; /* Stash at least npurge pages. */ - for (runselm = qr_next(&arena->runs_dirty, rd_link), + for (rdelm = qr_next(&arena->runs_dirty, rd_link), chunkselm = qr_next(&arena->chunks_cache, cc_link); - runselm != &arena->runs_dirty; runselm = runselm_next) { + rdelm != &arena->runs_dirty; rdelm = rdelm_next) { size_t npages; - runselm_next = qr_next(runselm, rd_link); + rdelm_next = qr_next(rdelm, rd_link); - if (runselm == &chunkselm->runs_dirty) { + if (rdelm == &chunkselm->rdelm) { extent_node_t *chunkselm_next; bool zero; UNUSED void *chunk; @@ -1144,9 +1145,11 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, chunkselm = chunkselm_next; } else { arena_chunk_t *chunk = - (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); - size_t pageind = arena_miscelm_to_pageind(runselm); - arena_run_t *run = &runselm->run; + (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + size_t pageind = arena_miscelm_to_pageind(miscelm); + arena_run_t *run = &miscelm->run; size_t run_size = arena_mapbits_unallocated_size_get(chunk, pageind); @@ -1167,12 +1170,12 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, arena_run_split_large(arena, run, run_size, false); /* Stash. */ if (false) - qr_new(runselm, rd_link); /* Redundant. */ + qr_new(rdelm, rd_link); /* Redundant. */ else { - assert(qr_next(runselm, rd_link) == runselm); - assert(qr_prev(runselm, rd_link) == runselm); + assert(qr_next(rdelm, rd_link) == rdelm); + assert(qr_prev(rdelm, rd_link) == rdelm); } - qr_meld(purge_runs_sentinel, runselm, rd_link); + qr_meld(purge_runs_sentinel, rdelm, rd_link); } nstashed += npages; @@ -1184,11 +1187,12 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, } static size_t -arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, +arena_purge_stashed(arena_t *arena, + arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { size_t npurged, nmadvise; - arena_chunk_map_misc_t *runselm; + arena_runs_dirty_link_t *rdelm; extent_node_t *chunkselm; if (config_stats) @@ -1196,13 +1200,12 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, npurged = 0; malloc_mutex_unlock(&arena->lock); - for (runselm = qr_next(purge_runs_sentinel, rd_link), + for (rdelm = qr_next(purge_runs_sentinel, rd_link), chunkselm = qr_next(purge_chunks_sentinel, cc_link); - runselm != purge_runs_sentinel; runselm = qr_next(runselm, - rd_link)) { + rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { size_t npages; - if (runselm == &chunkselm->runs_dirty) { + if (rdelm == &chunkselm->rdelm) { size_t size = extent_node_size_get(chunkselm); bool unzeroed; @@ -1216,8 +1219,10 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, size_t pageind, run_size, flag_unzeroed, i; bool unzeroed; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(runselm); - pageind = arena_miscelm_to_pageind(runselm); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + pageind = arena_miscelm_to_pageind(miscelm); run_size = arena_mapbits_large_size_get(chunk, pageind); npages = run_size >> LG_PAGE; @@ -1259,18 +1264,18 @@ arena_purge_stashed(arena_t *arena, arena_chunk_map_misc_t *purge_runs_sentinel, static void arena_unstash_purged(arena_t *arena, - arena_chunk_map_misc_t *purge_runs_sentinel, + arena_runs_dirty_link_t *purge_runs_sentinel, extent_node_t *purge_chunks_sentinel) { - arena_chunk_map_misc_t *runselm, *runselm_next; + arena_runs_dirty_link_t *rdelm, *rdelm_next; extent_node_t *chunkselm; /* Deallocate runs. */ - for (runselm = qr_next(purge_runs_sentinel, rd_link), + for (rdelm = qr_next(purge_runs_sentinel, rd_link), chunkselm = qr_next(purge_chunks_sentinel, cc_link); - runselm != purge_runs_sentinel; runselm = runselm_next) { - runselm_next = qr_next(runselm, rd_link); - if (runselm == &chunkselm->runs_dirty) { + rdelm != purge_runs_sentinel; rdelm = rdelm_next) { + rdelm_next = qr_next(rdelm, rd_link); + if (rdelm == &chunkselm->rdelm) { extent_node_t *chunkselm_next = qr_next(chunkselm, cc_link); void *addr = extent_node_addr_get(chunkselm); @@ -1281,8 +1286,10 @@ arena_unstash_purged(arena_t *arena, chunkselm = chunkselm_next; chunk_dalloc_arena(arena, addr, size, zeroed); } else { - arena_run_t *run = &runselm->run; - qr_remove(runselm, rd_link); + arena_chunk_map_misc_t *miscelm = + arena_rd_to_miscelm(rdelm); + arena_run_t *run = &miscelm->run; + qr_remove(rdelm, rd_link); arena_run_dalloc(arena, run, false, true); } } @@ -1292,7 +1299,7 @@ void arena_purge(arena_t *arena, bool all) { size_t npurge, npurgeable, npurged; - arena_chunk_map_misc_t purge_runs_sentinel; + arena_runs_dirty_link_t purge_runs_sentinel; extent_node_t purge_chunks_sentinel; /* From f5c8f37259d7697c3f850ac1e5ef63b724cf7689 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 10 Mar 2015 18:29:49 -0700 Subject: [PATCH 251/352] Normalize rdelm/rd structure field naming. --- include/jemalloc/internal/arena.h | 38 +++++++++++++++--------------- include/jemalloc/internal/extent.h | 8 +++---- src/arena.c | 8 +++---- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index de298e55..9cbc591a 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -329,25 +329,25 @@ struct arena_s { * * LRU-----------------------------------------------------------MRU * - * /------------------\ - * | arena | - * | | - * | /------------\ | /-----------\ - * ...---->|chunks_cache|<---------------------->| chunk |<--... - * | \------------/ | | | - * | | | | - * | | /---\ /---\ | | - * | | |run| |run| | | - * | | | | | | | | - * | /----------\ | |---| |---| | /-----\ | - * ...----->|runs_dirty|<---->|rd |<---->|rd |<---->|rdelm|<-----... - * | \----------/ | |---| |---| | \-----/ | - * | | | | | | | | - * | | | | | | | | - * | | \---/ \---/ | | - * | | | | - * | | | | - * \------------------/ \-----------/ + * /-- arena ---\ + * | | + * | | + * |------------| /- chunk -\ + * ...->|chunks_cache|<--------------------------->| /----\ |<--... + * |------------| | |node| | + * | | | | | | + * | | /- run -\ /- run -\ | | | | + * | | | | | | | | | | + * | | | | | | | | | | + * |------------| |-------| |-------| | |----| | + * ...->|runs_dirty |<-->|rd |<-->|rd |<---->|rd |<----... + * |------------| |-------| |-------| | |----| | + * | | | | | | | | | | + * | | | | | | | \----/ | + * | | \-------/ \-------/ | | + * | | | | + * | | | | + * \------------/ \---------/ */ arena_runs_dirty_link_t runs_dirty; extent_node_t chunks_cache; diff --git a/include/jemalloc/internal/extent.h b/include/jemalloc/internal/extent.h index 5dbc04a5..3751adc4 100644 --- a/include/jemalloc/internal/extent.h +++ b/include/jemalloc/internal/extent.h @@ -34,7 +34,7 @@ struct extent_node_s { prof_tctx_t *en_prof_tctx; /* Linkage for arena's runs_dirty and chunks_cache rings. */ - arena_runs_dirty_link_t rdelm; + arena_runs_dirty_link_t rd; qr(extent_node_t) cc_link; union { @@ -186,7 +186,7 @@ JEMALLOC_INLINE void extent_node_dirty_linkage_init(extent_node_t *node) { - qr_new(&node->rdelm, rd_link); + qr_new(&node->rd, rd_link); qr_new(node, cc_link); } @@ -195,7 +195,7 @@ extent_node_dirty_insert(extent_node_t *node, arena_runs_dirty_link_t *runs_dirty, extent_node_t *chunks_dirty) { - qr_meld(runs_dirty, &node->rdelm, rd_link); + qr_meld(runs_dirty, &node->rd, rd_link); qr_meld(chunks_dirty, node, cc_link); } @@ -203,7 +203,7 @@ JEMALLOC_INLINE void extent_node_dirty_remove(extent_node_t *node) { - qr_remove(&node->rdelm, rd_link); + qr_remove(&node->rd, rd_link); qr_remove(node, cc_link); } diff --git a/src/arena.c b/src/arena.c index 5d792f99..8af1a5df 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1064,7 +1064,7 @@ arena_dirty_count(arena_t *arena) rdelm != &arena->runs_dirty; rdelm = qr_next(rdelm, rd_link)) { size_t npages; - if (rdelm == &chunkselm->rdelm) { + if (rdelm == &chunkselm->rd) { npages = extent_node_size_get(chunkselm) >> LG_PAGE; chunkselm = qr_next(chunkselm, cc_link); } else { @@ -1122,7 +1122,7 @@ arena_stash_dirty(arena_t *arena, bool all, size_t npurge, size_t npages; rdelm_next = qr_next(rdelm, rd_link); - if (rdelm == &chunkselm->rdelm) { + if (rdelm == &chunkselm->rd) { extent_node_t *chunkselm_next; bool zero; UNUSED void *chunk; @@ -1205,7 +1205,7 @@ arena_purge_stashed(arena_t *arena, rdelm != purge_runs_sentinel; rdelm = qr_next(rdelm, rd_link)) { size_t npages; - if (rdelm == &chunkselm->rdelm) { + if (rdelm == &chunkselm->rd) { size_t size = extent_node_size_get(chunkselm); bool unzeroed; @@ -1275,7 +1275,7 @@ arena_unstash_purged(arena_t *arena, chunkselm = qr_next(purge_chunks_sentinel, cc_link); rdelm != purge_runs_sentinel; rdelm = rdelm_next) { rdelm_next = qr_next(rdelm, rd_link); - if (rdelm == &chunkselm->rdelm) { + if (rdelm == &chunkselm->rd) { extent_node_t *chunkselm_next = qr_next(chunkselm, cc_link); void *addr = extent_node_addr_get(chunkselm); From bc45d41d23bac598dbd38e5aac5a85b43d24bc04 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 11 Mar 2015 16:50:40 -0700 Subject: [PATCH 252/352] Fix a declaration-after-statement regression. --- src/arena.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/arena.c b/src/arena.c index 8af1a5df..e36cb502 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1215,11 +1215,10 @@ arena_purge_stashed(arena_t *arena, extent_node_zeroed_set(chunkselm, !unzeroed); chunkselm = qr_next(chunkselm, cc_link); } else { - arena_chunk_t *chunk; size_t pageind, run_size, flag_unzeroed, i; bool unzeroed; - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(rdelm); + arena_chunk_t *chunk = (arena_chunk_t + *)CHUNK_ADDR2BASE(rdelm); arena_chunk_map_misc_t *miscelm = arena_rd_to_miscelm(rdelm); pageind = arena_miscelm_to_pageind(miscelm); From fbd8d773ad0230ffba4e2c296dac3edcac9ca27e Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 11 Mar 2015 23:14:50 -0700 Subject: [PATCH 253/352] Fix unsigned comparison underflow. These bugs only affected tests and debug builds. --- include/jemalloc/internal/rtree.h | 2 +- src/rtree.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/jemalloc/internal/rtree.h b/include/jemalloc/internal/rtree.h index 2eb726d6..c1fb90c4 100644 --- a/include/jemalloc/internal/rtree.h +++ b/include/jemalloc/internal/rtree.h @@ -260,7 +260,7 @@ rtree_set(rtree_t *rtree, uintptr_t key, const extent_node_t *val) rtree_val_write(rtree, &node[subkey], val); return (false); } - assert(i < rtree->height - 1); + assert(i + 1 < rtree->height); child = rtree_child_read(rtree, &node[subkey], i); if (child == NULL) return (true); diff --git a/src/rtree.c b/src/rtree.c index 47d9084e..af0d97e7 100644 --- a/src/rtree.c +++ b/src/rtree.c @@ -63,7 +63,7 @@ static void rtree_delete_subtree(rtree_t *rtree, rtree_node_elm_t *node, unsigned level) { - if (level < rtree->height - 1) { + if (level + 1 < rtree->height) { size_t nchildren, i; nchildren = ZU(1) << rtree->levels[level].bits; From d69964bd2d31387f79a5f0494de8fd255b693afb Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Thu, 12 Mar 2015 16:25:18 -0700 Subject: [PATCH 254/352] Fix a heap profiling regression. Fix prof_tctx_comp() to incorporate tctx state into the comparison. During a dump it is possible for both a purgatory tctx and an otherwise equivalent nominal tctx to reside in the tree at the same time. This regression was introduced by 602c8e0971160e4b85b08b16cf8a2375aa24bc04 (Implement per thread heap profiling.). --- src/prof.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/prof.c b/src/prof.c index 4f1580b0..84fa5fda 100644 --- a/src/prof.c +++ b/src/prof.c @@ -137,8 +137,13 @@ prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) { uint64_t a_uid = a->thr_uid; uint64_t b_uid = b->thr_uid; - - return ((a_uid > b_uid) - (a_uid < b_uid)); + int ret = (a_uid > b_uid) - (a_uid < b_uid); + if (ret == 0) { + prof_tctx_state_t a_state = a->state; + prof_tctx_state_t b_state = b->state; + ret = (a_state > b_state) - (a_state < b_state); + } + return (ret); } rb_gen(static UNUSED, tctx_tree_, prof_tctx_tree_t, prof_tctx_t, From f69e2f6fdab40c7612be5fd69960b8c7d40dba44 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Thu, 12 Mar 2015 08:51:05 +0900 Subject: [PATCH 255/352] Use the error code given to buferror on Windows a14bce85 made buferror not take an error code, and make the Windows code path for buferror use GetLastError, while the alternative code paths used errno. Then 2a83ed02 made buferror take an error code again, and while it changed the non-Windows code paths to use that error code, the Windows code path was not changed accordingly. --- src/util.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util.c b/src/util.c index a964d700..a6ef5d50 100644 --- a/src/util.c +++ b/src/util.c @@ -81,7 +81,7 @@ buferror(int err, char *buf, size_t buflen) { #ifdef _WIN32 - FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0, + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, (LPSTR)buf, buflen, NULL); return (0); #elif defined(__GLIBC__) && defined(_GNU_SOURCE) From d6384b09e137874d7cdf527e5bb50abba0ae5f95 Mon Sep 17 00:00:00 2001 From: Daniel Micay Date: Thu, 16 Oct 2014 17:02:18 -0400 Subject: [PATCH 256/352] use CLOCK_MONOTONIC in the timer if it's available Linux sets _POSIX_MONOTONIC_CLOCK to 0 meaning it *might* be available, so a sysconf check is necessary at runtime with a fallback to the mandatory CLOCK_REALTIME clock. --- test/include/test/timer.h | 10 ++++++++++ test/src/timer.c | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/test/include/test/timer.h b/test/include/test/timer.h index 496072ac..9ffbaef5 100644 --- a/test/include/test/timer.h +++ b/test/include/test/timer.h @@ -1,10 +1,20 @@ /* Simple timer, for use in benchmark reporting. */ +#include #include +#define JEMALLOC_CLOCK_GETTIME defined(_POSIX_MONOTONIC_CLOCK) \ + && _POSIX_MONOTONIC_CLOCK >= 0 + typedef struct { +#if JEMALLOC_CLOCK_GETTIME + struct timespec tv0; + struct timespec tv1; + int clock_id; +#else struct timeval tv0; struct timeval tv1; +#endif } timedelta_t; void timer_start(timedelta_t *timer); diff --git a/test/src/timer.c b/test/src/timer.c index 36fbedd4..338a9eff 100644 --- a/test/src/timer.c +++ b/test/src/timer.c @@ -4,22 +4,39 @@ void timer_start(timedelta_t *timer) { +#if JEMALLOC_CLOCK_GETTIME + if (sysconf(_SC_MONOTONIC_CLOCK) <= 0) + timer->clock_id = CLOCK_REALTIME; + else + timer->clock_id = CLOCK_MONOTONIC; + clock_gettime(timer->clock_id, &timer->tv0); +#else gettimeofday(&timer->tv0, NULL); +#endif } void timer_stop(timedelta_t *timer) { +#if JEMALLOC_CLOCK_GETTIME + clock_gettime(timer->clock_id, &timer->tv1); +#else gettimeofday(&timer->tv1, NULL); +#endif } uint64_t timer_usec(const timedelta_t *timer) { +#if JEMALLOC_CLOCK_GETTIME + return (((timer->tv1.tv_sec - timer->tv0.tv_sec) * 1000000) + + (timer->tv1.tv_nsec - timer->tv0.tv_nsec) / 1000); +#else return (((timer->tv1.tv_sec - timer->tv0.tv_sec) * 1000000) + timer->tv1.tv_usec - timer->tv0.tv_usec); +#endif } void From 764b00023f2bc97f240c3a758ed23ce9c0ad8526 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 14 Mar 2015 14:01:35 -0700 Subject: [PATCH 257/352] Fix a heap profiling regression. Add the prof_tctx_state_destroying transitionary state to fix a race between a thread destroying a tctx and another thread creating a new equivalent tctx. This regression was introduced by 602c8e0971160e4b85b08b16cf8a2375aa24bc04 (Implement per thread heap profiling.). --- include/jemalloc/internal/prof.h | 1 + src/prof.c | 44 ++++++++++++++++++++++---------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index f5082438..8967333a 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -81,6 +81,7 @@ struct prof_cnt_s { typedef enum { prof_tctx_state_initializing, prof_tctx_state_nominal, + prof_tctx_state_destroying, prof_tctx_state_dumping, prof_tctx_state_purgatory /* Dumper must finish destroying. */ } prof_tctx_state_t; diff --git a/src/prof.c b/src/prof.c index 84fa5fda..e86669c2 100644 --- a/src/prof.c +++ b/src/prof.c @@ -642,10 +642,13 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); destroy_tdata = prof_tdata_should_destroy(tdata, false); + if (tctx->state == prof_tctx_state_nominal) + tctx->state = prof_tctx_state_destroying; malloc_mutex_unlock(tdata->lock); malloc_mutex_lock(gctx->lock); - if (tctx->state != prof_tctx_state_dumping) { + switch (tctx->state) { + case prof_tctx_state_destroying: tctx_tree_remove(&gctx->tctxs, tctx); destroy_tctx = true; if (prof_gctx_should_destroy(gctx)) { @@ -667,7 +670,8 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) destroy_gctx = true; } else destroy_gctx = false; - } else { + break; + case prof_tctx_state_dumping: /* * A dumping thread needs tctx to remain valid until dumping * has finished. Change state such that the dumping thread will @@ -676,6 +680,9 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) tctx->state = prof_tctx_state_purgatory; destroy_tctx = false; destroy_gctx = false; + break; + default: + not_reached(); } malloc_mutex_unlock(gctx->lock); if (destroy_gctx) { @@ -1021,21 +1028,30 @@ prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata) { malloc_mutex_lock(tctx->gctx->lock); - if (tctx->state == prof_tctx_state_initializing) { + + switch (tctx->state) { + case prof_tctx_state_initializing: + case prof_tctx_state_destroying: malloc_mutex_unlock(tctx->gctx->lock); return; - } - assert(tctx->state == prof_tctx_state_nominal); - tctx->state = prof_tctx_state_dumping; - malloc_mutex_unlock(tctx->gctx->lock); + case prof_tctx_state_nominal: + tctx->state = prof_tctx_state_dumping; + malloc_mutex_unlock(tctx->gctx->lock); - memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); + memcpy(&tctx->dump_cnts, &tctx->cnts, sizeof(prof_cnt_t)); - tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; - tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; - if (opt_prof_accum) { - tdata->cnt_summed.accumobjs += tctx->dump_cnts.accumobjs; - tdata->cnt_summed.accumbytes += tctx->dump_cnts.accumbytes; + tdata->cnt_summed.curobjs += tctx->dump_cnts.curobjs; + tdata->cnt_summed.curbytes += tctx->dump_cnts.curbytes; + if (opt_prof_accum) { + tdata->cnt_summed.accumobjs += + tctx->dump_cnts.accumobjs; + tdata->cnt_summed.accumbytes += + tctx->dump_cnts.accumbytes; + } + break; + case prof_tctx_state_dumping: + case prof_tctx_state_purgatory: + not_reached(); } } @@ -1059,6 +1075,7 @@ prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) switch (tctx->state) { case prof_tctx_state_nominal: + case prof_tctx_state_destroying: /* New since dumping started; ignore. */ break; case prof_tctx_state_dumping: @@ -1094,6 +1111,7 @@ prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) switch (tctx->state) { case prof_tctx_state_nominal: + case prof_tctx_state_destroying: /* New since dumping started; ignore. */ break; case prof_tctx_state_dumping: From 262146dfc4778f0671ab86458acd4ec531a80a34 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Sat, 14 Mar 2015 14:34:16 -0700 Subject: [PATCH 258/352] Eliminate innocuous compiler warnings. --- src/prof.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/prof.c b/src/prof.c index e86669c2..e9daa6f8 100644 --- a/src/prof.c +++ b/src/prof.c @@ -683,6 +683,8 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) break; default: not_reached(); + destroy_tctx = false; + destroy_gctx = false; } malloc_mutex_unlock(gctx->lock); if (destroy_gctx) { From 04211e226628c41da4b3804ba411b5dd4b3a02ab Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 16 Mar 2015 15:11:06 -0700 Subject: [PATCH 259/352] Fix heap profiling regressions. Remove the prof_tctx_state_destroying transitory state and instead add the tctx_uid field, so that the tuple uniquely identifies a tctx. This assures that tctx's are well ordered even when more than two with the same thr_uid coexist. A previous attempted fix based on prof_tctx_state_destroying was only sufficient for protecting against two coexisting tctx's, but it also introduced a new dumping race. These regressions were introduced by 602c8e0971160e4b85b08b16cf8a2375aa24bc04 (Implement per thread heap profiling.) and 764b00023f2bc97f240c3a758ed23ce9c0ad8526 (Fix a heap profiling regression.). --- include/jemalloc/internal/prof.h | 23 ++++++++++++++++++++++- src/prof.c | 21 +++++++++------------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/include/jemalloc/internal/prof.h b/include/jemalloc/internal/prof.h index 8967333a..2e227116 100644 --- a/include/jemalloc/internal/prof.h +++ b/include/jemalloc/internal/prof.h @@ -81,7 +81,6 @@ struct prof_cnt_s { typedef enum { prof_tctx_state_initializing, prof_tctx_state_nominal, - prof_tctx_state_destroying, prof_tctx_state_dumping, prof_tctx_state_purgatory /* Dumper must finish destroying. */ } prof_tctx_state_t; @@ -102,6 +101,21 @@ struct prof_tctx_s { /* Associated global context. */ prof_gctx_t *gctx; + /* + * UID that distinguishes multiple tctx's created by the same thread, + * but coexisting in gctx->tctxs. There are two ways that such + * coexistence can occur: + * - A dumper thread can cause a tctx to be retained in the purgatory + * state. + * - Although a single "producer" thread must create all tctx's which + * share the same thr_uid, multiple "consumers" can each concurrently + * execute portions of prof_tctx_destroy(). prof_tctx_destroy() only + * gets called once each time cnts.cur{objs,bytes} drop to 0, but this + * threshold can be hit again before the first consumer finishes + * executing prof_tctx_destroy(). + */ + uint64_t tctx_uid; + /* Linkage into gctx's tctxs. */ rb_node(prof_tctx_t) tctx_link; @@ -178,6 +192,13 @@ struct prof_tdata_s { rb_node(prof_tdata_t) tdata_link; + /* + * Counter used to initialize prof_tctx_t's tctx_uid. No locking is + * necessary when incrementing this field, because only one thread ever + * does so. + */ + uint64_t tctx_uid_next; + /* * Hash of (prof_bt_t *)-->(prof_tctx_t *). Each thread tracks * backtraces for which it has non-zero allocation/deallocation counters diff --git a/src/prof.c b/src/prof.c index e9daa6f8..f2a37253 100644 --- a/src/prof.c +++ b/src/prof.c @@ -135,13 +135,13 @@ static char *prof_thread_name_alloc(tsd_t *tsd, const char *thread_name); JEMALLOC_INLINE_C int prof_tctx_comp(const prof_tctx_t *a, const prof_tctx_t *b) { - uint64_t a_uid = a->thr_uid; - uint64_t b_uid = b->thr_uid; - int ret = (a_uid > b_uid) - (a_uid < b_uid); + uint64_t a_thr_uid = a->thr_uid; + uint64_t b_thr_uid = b->thr_uid; + int ret = (a_thr_uid > b_thr_uid) - (a_thr_uid < b_thr_uid); if (ret == 0) { - prof_tctx_state_t a_state = a->state; - prof_tctx_state_t b_state = b->state; - ret = (a_state > b_state) - (a_state < b_state); + uint64_t a_tctx_uid = a->tctx_uid; + uint64_t b_tctx_uid = b->tctx_uid; + ret = (a_tctx_uid > b_tctx_uid) - (a_tctx_uid < b_tctx_uid); } return (ret); } @@ -642,13 +642,11 @@ prof_tctx_destroy(tsd_t *tsd, prof_tctx_t *tctx) ckh_remove(tsd, &tdata->bt2tctx, &gctx->bt, NULL, NULL); destroy_tdata = prof_tdata_should_destroy(tdata, false); - if (tctx->state == prof_tctx_state_nominal) - tctx->state = prof_tctx_state_destroying; malloc_mutex_unlock(tdata->lock); malloc_mutex_lock(gctx->lock); switch (tctx->state) { - case prof_tctx_state_destroying: + case prof_tctx_state_nominal: tctx_tree_remove(&gctx->tctxs, tctx); destroy_tctx = true; if (prof_gctx_should_destroy(gctx)) { @@ -795,6 +793,7 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) ret.p->thr_uid = tdata->thr_uid; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); ret.p->gctx = gctx; + ret.p->tctx_uid = tdata->tctx_uid_next++; ret.p->prepared = true; ret.p->state = prof_tctx_state_initializing; malloc_mutex_lock(tdata->lock); @@ -1033,7 +1032,6 @@ prof_tctx_merge_tdata(prof_tctx_t *tctx, prof_tdata_t *tdata) switch (tctx->state) { case prof_tctx_state_initializing: - case prof_tctx_state_destroying: malloc_mutex_unlock(tctx->gctx->lock); return; case prof_tctx_state_nominal: @@ -1077,7 +1075,6 @@ prof_tctx_merge_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) switch (tctx->state) { case prof_tctx_state_nominal: - case prof_tctx_state_destroying: /* New since dumping started; ignore. */ break; case prof_tctx_state_dumping: @@ -1113,7 +1110,6 @@ prof_tctx_finish_iter(prof_tctx_tree_t *tctxs, prof_tctx_t *tctx, void *arg) switch (tctx->state) { case prof_tctx_state_nominal: - case prof_tctx_state_destroying: /* New since dumping started; ignore. */ break; case prof_tctx_state_dumping: @@ -1690,6 +1686,7 @@ prof_tdata_init_impl(tsd_t *tsd, uint64_t thr_uid, uint64_t thr_discrim, tdata->thread_name = thread_name; tdata->attached = true; tdata->expired = false; + tdata->tctx_uid_next = 0; if (ckh_new(tsd, &tdata->bt2tctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) { From c9db461ffb608ad32aed0e34663ae58a992e1003 Mon Sep 17 00:00:00 2001 From: Mike Hommey Date: Tue, 17 Mar 2015 12:09:30 +0900 Subject: [PATCH 260/352] Use InterlockedCompareExchange instead of non-existing InterlockedCompareExchange32 --- include/jemalloc/internal/atomic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/jemalloc/internal/atomic.h b/include/jemalloc/internal/atomic.h index 0d33065e..522dd2ae 100644 --- a/include/jemalloc/internal/atomic.h +++ b/include/jemalloc/internal/atomic.h @@ -457,7 +457,7 @@ atomic_cas_uint32(uint32_t *p, uint32_t c, uint32_t s) { uint32_t o; - o = InterlockedCompareExchange32(p, s, c); + o = InterlockedCompareExchange(p, s, c); return (o != c); } From 8d6a3e8321a7767cb2ca0930b85d5d488a8cc659 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 18 Mar 2015 18:55:33 -0700 Subject: [PATCH 261/352] Implement dynamic per arena control over dirty page purging. Add mallctls: - arenas.lg_dirty_mult is initialized via opt.lg_dirty_mult, and can be modified to change the initial lg_dirty_mult setting for newly created arenas. - arena..lg_dirty_mult controls an individual arena's dirty page purging threshold, and synchronously triggers any purging that may be necessary to maintain the constraint. - arena..chunk.purge allows the per arena dirty page purging function to be replaced. This resolves #93. --- doc/jemalloc.xml.in | 88 +++++++++++-- include/jemalloc/internal/arena.h | 16 ++- include/jemalloc/internal/chunk.h | 6 + include/jemalloc/internal/private_symbols.txt | 7 + include/jemalloc/jemalloc_typedefs.h.in | 1 + src/arena.c | 87 +++++++++++-- src/chunk.c | 37 +++++- src/ctl.c | 121 ++++++++++++------ src/huge.c | 38 ++++-- src/stats.c | 10 ++ test/integration/chunk.c | 72 ++++++++--- test/unit/mallctl.c | 66 ++++++++++ test/unit/rtree.c | 10 +- 13 files changed, 460 insertions(+), 99 deletions(-) diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 747e03f4..01ac38c3 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -937,7 +937,11 @@ for (i = 0; i < nbins; i++) { provides the kernel with sufficient information to recycle dirty pages if physical memory becomes scarce and the pages remain unused. The default minimum ratio is 8:1 (2^3:1); an option value of -1 will - disable dirty page purging. + disable dirty page purging. See arenas.lg_dirty_mult + and arena.<i>.lg_dirty_mult + for related dynamic control options. @@ -1151,7 +1155,7 @@ malloc_conf = "xmalloc:true";]]> opt.prof_active (bool) - rw + r- [] Profiling activated/deactivated. This is a secondary @@ -1489,6 +1493,20 @@ malloc_conf = "xmalloc:true";]]> settings. + + + arena.<i>.lg_dirty_mult + (ssize_t) + rw + + Current per-arena minimum ratio (log base 2) of active + to dirty pages for arena <i>. Each time this interface is set and + the ratio is increased, pages are synchronously purged as necessary to + impose the new ratio. See opt.lg_dirty_mult + for additional information. + + arena.<i>.chunk.alloc @@ -1544,12 +1562,12 @@ malloc_conf = "xmalloc:true";]]> allocation for arenas created via arenas.extend such that all chunks originate from an application-supplied chunk allocator - (by setting custom chunk allocation/deallocation functions just after - arena creation), but the automatically created arenas may have already - created chunks prior to the application having an opportunity to take - over chunk allocation. + (by setting custom chunk allocation/deallocation/purge functions just + after arena creation), but the automatically created arenas may have + already created chunks prior to the application having an opportunity to + take over chunk allocation. - typedef void (chunk_dalloc_t) + typedef bool (chunk_dalloc_t) void *chunk size_t size unsigned arena_ind @@ -1557,7 +1575,47 @@ malloc_conf = "xmalloc:true";]]> A chunk deallocation function conforms to the chunk_dalloc_t type and deallocates a chunk of given size on - behalf of arena arena_ind. + behalf of arena arena_ind, returning false upon + success. + + + + + arena.<i>.chunk.purge + (chunk_purge_t *) + rw + + Get or set the chunk purge function for arena <i>. + A chunk purge function optionally discards physical pages associated + with pages in the chunk's virtual memory range but leaves the virtual + memory mapping intact, and indicates via its return value whether pages + in the virtual memory range will be zero-filled the next time they are + accessed. If setting, the chunk purge function must be capable of + purging all extant chunks associated with arena <i>, usually by + passing unknown chunks to the purge function that was replaced. In + practice, it is feasible to control allocation for arenas created via + arenas.extend + such that all chunks originate from an application-supplied chunk + allocator (by setting custom chunk allocation/deallocation/purge + functions just after arena creation), but the automatically created + arenas may have already created chunks prior to the application having + an opportunity to take over chunk allocation. + + typedef bool (chunk_purge_t) + void *chunk + size_t offset + size_t length + unsigned arena_ind + + A chunk purge function conforms to the chunk_purge_t type + and purges pages within chunk at + offset bytes, extending for + length on behalf of arena + arena_ind, returning false if pages within the + purged virtual memory range will be zero-filled the next time they are + accessed. Note that the memory range being purged may span multiple + contiguous chunks, e.g. when purging memory that backed a huge + allocation. @@ -1581,6 +1639,20 @@ malloc_conf = "xmalloc:true";]]> initialized. + + + arenas.lg_dirty_mult + (ssize_t) + rw + + Current default per-arena minimum ratio (log base 2) of + active to dirty pages, used to initialize arena.<i>.lg_dirty_mult + during arena creation. See opt.lg_dirty_mult + for additional information. + + arenas.quantum diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 9cbc591a..56ee74aa 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -16,10 +16,10 @@ /* * The minimum ratio of active:dirty pages per arena is computed as: * - * (nactive >> opt_lg_dirty_mult) >= ndirty + * (nactive >> lg_dirty_mult) >= ndirty * - * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times - * as many active pages as dirty pages. + * So, supposing that lg_dirty_mult is 3, there can be no less than 8 times as + * many active pages as dirty pages. */ #define LG_DIRTY_MULT_DEFAULT 3 @@ -304,6 +304,9 @@ struct arena_s { */ arena_chunk_t *spare; + /* Minimum ratio (log base 2) of nactive:ndirty. */ + ssize_t lg_dirty_mult; + /* Number of pages in active runs and huge regions. */ size_t nactive; @@ -376,10 +379,11 @@ struct arena_s { malloc_mutex_t node_cache_mtx; /* - * User-configurable chunk allocation and deallocation functions. + * User-configurable chunk allocation/deallocation/purge functions. */ chunk_alloc_t *chunk_alloc; chunk_dalloc_t *chunk_dalloc; + chunk_purge_t *chunk_purge; /* bins is used to store trees of free regions. */ arena_bin_t bins[NBINS]; @@ -416,6 +420,8 @@ void arena_chunk_ralloc_huge_shrink(arena_t *arena, void *chunk, size_t oldsize, size_t usize); bool arena_chunk_ralloc_huge_expand(arena_t *arena, void *chunk, size_t oldsize, size_t usize, bool *zero); +ssize_t arena_lg_dirty_mult_get(arena_t *arena); +bool arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult); void arena_maybe_purge(arena_t *arena); void arena_purge_all(arena_t *arena); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, @@ -462,6 +468,8 @@ void *arena_ralloc(tsd_t *tsd, arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, tcache_t *tcache); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); +ssize_t arena_lg_dirty_mult_default_get(void); +bool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult); void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats); diff --git a/include/jemalloc/internal/chunk.h b/include/jemalloc/internal/chunk.h index 1af5b24b..80938147 100644 --- a/include/jemalloc/internal/chunk.h +++ b/include/jemalloc/internal/chunk.h @@ -54,6 +54,12 @@ void chunk_dalloc_arena(arena_t *arena, void *chunk, size_t size, bool chunk_dalloc_default(void *chunk, size_t size, unsigned arena_ind); void chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc, void *chunk, size_t size); +bool chunk_purge_arena(arena_t *arena, void *chunk, size_t offset, + size_t length); +bool chunk_purge_default(void *chunk, size_t offset, size_t length, + unsigned arena_ind); +bool chunk_purge_wrapper(arena_t *arena, chunk_purge_t *chunk_purge, + void *chunk, size_t offset, size_t length); bool chunk_boot(void); void chunk_prefork(void); void chunk_postfork_parent(void); diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index d086db18..bc0f2a6a 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -30,6 +30,10 @@ arena_dalloc_small arena_dss_prec_get arena_dss_prec_set arena_init +arena_lg_dirty_mult_default_get +arena_lg_dirty_mult_default_set +arena_lg_dirty_mult_get +arena_lg_dirty_mult_set arena_malloc arena_malloc_large arena_malloc_small @@ -151,6 +155,9 @@ chunk_npages chunk_postfork_child chunk_postfork_parent chunk_prefork +chunk_purge_arena +chunk_purge_default +chunk_purge_wrapper chunk_record chunk_register chunks_rtree diff --git a/include/jemalloc/jemalloc_typedefs.h.in b/include/jemalloc/jemalloc_typedefs.h.in index 8092f1b1..d4b46908 100644 --- a/include/jemalloc/jemalloc_typedefs.h.in +++ b/include/jemalloc/jemalloc_typedefs.h.in @@ -1,2 +1,3 @@ typedef void *(chunk_alloc_t)(void *, size_t, size_t, bool *, unsigned); typedef bool (chunk_dalloc_t)(void *, size_t, unsigned); +typedef bool (chunk_purge_t)(void *, size_t, size_t, unsigned); diff --git a/src/arena.c b/src/arena.c index e36cb502..7272682d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -5,6 +5,7 @@ /* Data. */ ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; +static ssize_t lg_dirty_mult_default; arena_bin_info_t arena_bin_info[NBINS]; size_t map_bias; @@ -1032,15 +1033,49 @@ arena_run_alloc_small(arena_t *arena, size_t size, index_t binind) return (arena_run_alloc_small_helper(arena, size, binind)); } +static bool +arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult) +{ + + return (lg_dirty_mult >= -1 && lg_dirty_mult < (sizeof(size_t) << 3)); +} + +ssize_t +arena_lg_dirty_mult_get(arena_t *arena) +{ + ssize_t lg_dirty_mult; + + malloc_mutex_lock(&arena->lock); + lg_dirty_mult = arena->lg_dirty_mult; + malloc_mutex_unlock(&arena->lock); + + return (lg_dirty_mult); +} + +bool +arena_lg_dirty_mult_set(arena_t *arena, ssize_t lg_dirty_mult) +{ + + if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) + return (true); + + malloc_mutex_lock(&arena->lock); + arena->lg_dirty_mult = lg_dirty_mult; + arena_maybe_purge(arena); + malloc_mutex_unlock(&arena->lock); + + return (false); +} + void arena_maybe_purge(arena_t *arena) { size_t threshold; /* Don't purge if the option is disabled. */ - if (opt_lg_dirty_mult < 0) + if (arena->lg_dirty_mult < 0) return; - threshold = (arena->nactive >> opt_lg_dirty_mult); + threshold = (arena->nactive >> arena->lg_dirty_mult); threshold = threshold < chunk_npages ? chunk_npages : threshold; /* * Don't purge unless the number of purgeable pages exceeds the @@ -1096,7 +1131,7 @@ arena_compute_npurge(arena_t *arena, bool all) * purge. */ if (!all) { - size_t threshold = (arena->nactive >> opt_lg_dirty_mult); + size_t threshold = (arena->nactive >> arena->lg_dirty_mult); threshold = threshold < chunk_npages ? chunk_npages : threshold; npurge = arena->ndirty - threshold; @@ -1192,6 +1227,7 @@ arena_purge_stashed(arena_t *arena, extent_node_t *purge_chunks_sentinel) { size_t npurged, nmadvise; + chunk_purge_t *chunk_purge; arena_runs_dirty_link_t *rdelm; extent_node_t *chunkselm; @@ -1199,6 +1235,7 @@ arena_purge_stashed(arena_t *arena, nmadvise = 0; npurged = 0; + chunk_purge = arena->chunk_purge; malloc_mutex_unlock(&arena->lock); for (rdelm = qr_next(purge_runs_sentinel, rd_link), chunkselm = qr_next(purge_chunks_sentinel, cc_link); @@ -1207,11 +1244,16 @@ arena_purge_stashed(arena_t *arena, if (rdelm == &chunkselm->rd) { size_t size = extent_node_size_get(chunkselm); + void *addr, *chunk; + size_t offset; bool unzeroed; npages = size >> LG_PAGE; - unzeroed = pages_purge(extent_node_addr_get(chunkselm), - size); + addr = extent_node_addr_get(chunkselm); + chunk = CHUNK_ADDR2BASE(addr); + offset = CHUNK_ADDR2OFFSET(addr); + unzeroed = chunk_purge_wrapper(arena, chunk_purge, + chunk, offset, size); extent_node_zeroed_set(chunkselm, !unzeroed); chunkselm = qr_next(chunkselm, cc_link); } else { @@ -1226,15 +1268,15 @@ arena_purge_stashed(arena_t *arena, npages = run_size >> LG_PAGE; assert(pageind + npages <= chunk_npages); - unzeroed = pages_purge((void *)((uintptr_t)chunk + - (pageind << LG_PAGE)), run_size); + unzeroed = chunk_purge_wrapper(arena, chunk_purge, + chunk, pageind << LG_PAGE, run_size); flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; /* * Set the unzeroed flag for all pages, now that - * pages_purge() has returned whether the pages were - * zeroed as a side effect of purging. This chunk map - * modification is safe even though the arena mutex + * chunk_purge_wrapper() has returned whether the pages + * were zeroed as a side effect of purging. This chunk + * map modification is safe even though the arena mutex * isn't currently owned by this thread, because the run * is marked as allocated, thus protecting it from being * modified by any other thread. As long as these @@ -1294,7 +1336,7 @@ arena_unstash_purged(arena_t *arena, } } -void +static void arena_purge(arena_t *arena, bool all) { size_t npurge, npurgeable, npurged; @@ -1309,7 +1351,7 @@ arena_purge(arena_t *arena, bool all) size_t ndirty = arena_dirty_count(arena); assert(ndirty == arena->ndirty); } - assert((arena->nactive >> opt_lg_dirty_mult) < arena->ndirty || all); + assert((arena->nactive >> arena->lg_dirty_mult) < arena->ndirty || all); if (config_stats) arena->stats.npurge++; @@ -2596,6 +2638,23 @@ arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) return (false); } +ssize_t +arena_lg_dirty_mult_default_get(void) +{ + + return ((ssize_t)atomic_read_z((size_t *)&lg_dirty_mult_default)); +} + +bool +arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult) +{ + + if (!arena_lg_dirty_mult_valid(lg_dirty_mult)) + return (true); + atomic_write_z((size_t *)&lg_dirty_mult_default, (size_t)lg_dirty_mult); + return (false); +} + void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, @@ -2702,6 +2761,7 @@ arena_new(unsigned ind) arena->spare = NULL; + arena->lg_dirty_mult = arena_lg_dirty_mult_default_get(); arena->nactive = 0; arena->ndirty = 0; @@ -2727,6 +2787,7 @@ arena_new(unsigned ind) arena->chunk_alloc = chunk_alloc_default; arena->chunk_dalloc = chunk_dalloc_default; + arena->chunk_purge = chunk_purge_default; /* Initialize bins. */ for (i = 0; i < NBINS; i++) { @@ -2860,6 +2921,8 @@ arena_boot(void) size_t header_size; unsigned i; + arena_lg_dirty_mult_default_set(opt_lg_dirty_mult); + /* * Compute the header size such that it is large enough to contain the * page map. The page map is biased to omit entries for the header diff --git a/src/chunk.c b/src/chunk.c index fb8cd413..70634107 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -391,8 +391,10 @@ chunk_record(arena_t *arena, extent_tree_t *chunks_szad, * pages have already been purged, so that this is only * a virtual memory leak. */ - if (cache) - pages_purge(chunk, size); + if (cache) { + chunk_purge_wrapper(arena, arena->chunk_purge, + chunk, 0, size); + } goto label_return; } extent_node_init(node, arena, chunk, size, !unzeroed); @@ -485,6 +487,37 @@ chunk_dalloc_wrapper(arena_t *arena, chunk_dalloc_t *chunk_dalloc, void *chunk, JEMALLOC_VALGRIND_MAKE_MEM_NOACCESS(chunk, size); } +bool +chunk_purge_arena(arena_t *arena, void *chunk, size_t offset, size_t length) +{ + + assert(chunk != NULL); + assert(CHUNK_ADDR2BASE(chunk) == chunk); + assert((offset & PAGE_MASK) == 0); + assert(length != 0); + assert((length & PAGE_MASK) == 0); + + return (pages_purge((void *)((uintptr_t)chunk + (uintptr_t)offset), + length)); +} + +bool +chunk_purge_default(void *chunk, size_t offset, size_t length, + unsigned arena_ind) +{ + + return (chunk_purge_arena(chunk_arena_get(arena_ind), chunk, offset, + length)); +} + +bool +chunk_purge_wrapper(arena_t *arena, chunk_purge_t *chunk_purge, void *chunk, + size_t offset, size_t length) +{ + + return (chunk_purge(chunk, offset, length, arena->ind)); +} + static rtree_node_elm_t * chunks_rtree_node_alloc(size_t nelms) { diff --git a/src/ctl.c b/src/ctl.c index cd7927fc..447b8776 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -116,8 +116,10 @@ CTL_PROTO(tcache_destroy) CTL_PROTO(arena_i_purge) static void arena_purge(unsigned arena_ind); CTL_PROTO(arena_i_dss) +CTL_PROTO(arena_i_lg_dirty_mult) CTL_PROTO(arena_i_chunk_alloc) CTL_PROTO(arena_i_chunk_dalloc) +CTL_PROTO(arena_i_chunk_purge) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) @@ -129,6 +131,7 @@ CTL_PROTO(arenas_hchunk_i_size) INDEX_PROTO(arenas_hchunk_i) CTL_PROTO(arenas_narenas) CTL_PROTO(arenas_initialized) +CTL_PROTO(arenas_lg_dirty_mult) CTL_PROTO(arenas_quantum) CTL_PROTO(arenas_page) CTL_PROTO(arenas_tcache_max) @@ -283,12 +286,14 @@ static const ctl_named_node_t tcache_node[] = { static const ctl_named_node_t chunk_node[] = { {NAME("alloc"), CTL(arena_i_chunk_alloc)}, - {NAME("dalloc"), CTL(arena_i_chunk_dalloc)} + {NAME("dalloc"), CTL(arena_i_chunk_dalloc)}, + {NAME("purge"), CTL(arena_i_chunk_purge)} }; static const ctl_named_node_t arena_i_node[] = { {NAME("purge"), CTL(arena_i_purge)}, {NAME("dss"), CTL(arena_i_dss)}, + {NAME("lg_dirty_mult"), CTL(arena_i_lg_dirty_mult)}, {NAME("chunk"), CHILD(named, chunk)}, }; static const ctl_named_node_t super_arena_i_node[] = { @@ -337,6 +342,7 @@ static const ctl_indexed_node_t arenas_hchunk_node[] = { static const ctl_named_node_t arenas_node[] = { {NAME("narenas"), CTL(arenas_narenas)}, {NAME("initialized"), CTL(arenas_initialized)}, + {NAME("lg_dirty_mult"), CTL(arenas_lg_dirty_mult)}, {NAME("quantum"), CTL(arenas_quantum)}, {NAME("page"), CTL(arenas_page)}, {NAME("tcache_max"), CTL(arenas_tcache_max)}, @@ -1617,57 +1623,70 @@ label_return: } static int -arena_i_chunk_alloc_ctl(const size_t *mib, size_t miblen, void *oldp, +arena_i_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned arena_ind = mib[1]; arena_t *arena; - malloc_mutex_lock(&ctl_mtx); - if (arena_ind < narenas_total_get() && (arena = arena_get(tsd_fetch(), - arena_ind, false, true)) != NULL) { - malloc_mutex_lock(&arena->lock); - READ(arena->chunk_alloc, chunk_alloc_t *); - WRITE(arena->chunk_alloc, chunk_alloc_t *); - } else { + arena = arena_get(tsd_fetch(), arena_ind, false, (arena_ind == 0)); + if (arena == NULL) { ret = EFAULT; - goto label_outer_return; + goto label_return; } + + if (oldp != NULL && oldlenp != NULL) { + size_t oldval = arena_lg_dirty_mult_get(arena); + READ(oldval, ssize_t); + } + if (newp != NULL) { + if (newlen != sizeof(ssize_t)) { + ret = EINVAL; + goto label_return; + } + if (arena_lg_dirty_mult_set(arena, *(ssize_t *)newp)) { + ret = EFAULT; + goto label_return; + } + } + ret = 0; label_return: - malloc_mutex_unlock(&arena->lock); -label_outer_return: - malloc_mutex_unlock(&ctl_mtx); return (ret); } -static int -arena_i_chunk_dalloc_ctl(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - - int ret; - unsigned arena_ind = mib[1]; - arena_t *arena; - - malloc_mutex_lock(&ctl_mtx); - if (arena_ind < narenas_total_get() && (arena = arena_get(tsd_fetch(), - arena_ind, false, true)) != NULL) { - malloc_mutex_lock(&arena->lock); - READ(arena->chunk_dalloc, chunk_dalloc_t *); - WRITE(arena->chunk_dalloc, chunk_dalloc_t *); - } else { - ret = EFAULT; - goto label_outer_return; - } - ret = 0; -label_return: - malloc_mutex_unlock(&arena->lock); -label_outer_return: - malloc_mutex_unlock(&ctl_mtx); - return (ret); +#define CHUNK_FUNC(n) \ +static int \ +arena_i_chunk_##n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ + size_t *oldlenp, void *newp, size_t newlen) \ +{ \ + \ + int ret; \ + unsigned arena_ind = mib[1]; \ + arena_t *arena; \ + \ + malloc_mutex_lock(&ctl_mtx); \ + if (arena_ind < narenas_total_get() && (arena = \ + arena_get(tsd_fetch(), arena_ind, false, true)) != NULL) { \ + malloc_mutex_lock(&arena->lock); \ + READ(arena->chunk_##n, chunk_##n##_t *); \ + WRITE(arena->chunk_##n, chunk_##n##_t *); \ + } else { \ + ret = EFAULT; \ + goto label_outer_return; \ + } \ + ret = 0; \ +label_return: \ + malloc_mutex_unlock(&arena->lock); \ +label_outer_return: \ + malloc_mutex_unlock(&ctl_mtx); \ + return (ret); \ } +CHUNK_FUNC(alloc) +CHUNK_FUNC(dalloc) +CHUNK_FUNC(purge) +#undef CHUNK_FUNC static const ctl_named_node_t * arena_i_index(const size_t *mib, size_t miblen, size_t i) @@ -1736,6 +1755,32 @@ label_return: return (ret); } +static int +arenas_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + + if (oldp != NULL && oldlenp != NULL) { + size_t oldval = arena_lg_dirty_mult_default_get(); + READ(oldval, ssize_t); + } + if (newp != NULL) { + if (newlen != sizeof(ssize_t)) { + ret = EINVAL; + goto label_return; + } + if (arena_lg_dirty_mult_default_set(*(ssize_t *)newp)) { + ret = EFAULT; + goto label_return; + } + } + + ret = 0; +label_return: + return (ret); +} + CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) diff --git a/src/huge.c b/src/huge.c index 3092932e..aa26f5df 100644 --- a/src/huge.c +++ b/src/huge.c @@ -124,9 +124,10 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, size_t size, size_t extra, bool zero) { size_t usize_next; - bool zeroed; extent_node_t *node; arena_t *arena; + chunk_purge_t *chunk_purge; + bool zeroed; /* Increase usize to incorporate extra. */ while (usize < s2u(size+extra) && (usize_next = s2u(usize+1)) < oldsize) @@ -135,11 +136,18 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, if (oldsize == usize) return; + node = huge_node_get(ptr); + arena = extent_node_arena_get(node); + + malloc_mutex_lock(&arena->lock); + chunk_purge = arena->chunk_purge; + malloc_mutex_unlock(&arena->lock); + /* Fill if necessary (shrinking). */ if (oldsize > usize) { size_t sdiff = CHUNK_CEILING(usize) - usize; - zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + - usize), sdiff) : true; + zeroed = (sdiff != 0) ? !chunk_purge_wrapper(arena, chunk_purge, + CHUNK_ADDR2BASE(ptr), CHUNK_ADDR2OFFSET(ptr), usize) : true; if (config_fill && unlikely(opt_junk_free)) { memset((void *)((uintptr_t)ptr + usize), 0x5a, oldsize - usize); @@ -148,8 +156,6 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, } else zeroed = true; - node = huge_node_get(ptr); - arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ assert(extent_node_size_get(node) != usize); @@ -177,22 +183,29 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, static void huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) { - size_t sdiff; - bool zeroed; extent_node_t *node; arena_t *arena; + chunk_purge_t *chunk_purge; + size_t sdiff; + bool zeroed; + + node = huge_node_get(ptr); + arena = extent_node_arena_get(node); + + malloc_mutex_lock(&arena->lock); + chunk_purge = arena->chunk_purge; + malloc_mutex_unlock(&arena->lock); sdiff = CHUNK_CEILING(usize) - usize; - zeroed = (sdiff != 0) ? !pages_purge((void *)((uintptr_t)ptr + usize), - sdiff) : true; + zeroed = (sdiff != 0) ? !chunk_purge_wrapper(arena, chunk_purge, + CHUNK_ADDR2BASE((uintptr_t)ptr + usize), + CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff) : true; if (config_fill && unlikely(opt_junk_free)) { huge_dalloc_junk((void *)((uintptr_t)ptr + usize), oldsize - usize); zeroed = false; } - node = huge_node_get(ptr); - arena = extent_node_arena_get(node); malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ extent_node_size_set(node, usize); @@ -291,8 +304,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, } /* Attempt to expand the allocation in-place. */ - if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, - zero)) { + if (huge_ralloc_no_move_expand(ptr, oldsize, size + extra, zero)) { if (extra == 0) return (true); diff --git a/src/stats.c b/src/stats.c index e0f71651..f246c8bc 100644 --- a/src/stats.c +++ b/src/stats.c @@ -264,6 +264,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, { unsigned nthreads; const char *dss; + ssize_t lg_dirty_mult; size_t page, pactive, pdirty, mapped; size_t metadata_mapped, metadata_allocated; uint64_t npurge, nmadvise, purged; @@ -282,6 +283,15 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_I_GET("stats.arenas.0.dss", &dss, const char *); malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n", dss); + CTL_I_GET("stats.arenas.0.lg_dirty_mult", &lg_dirty_mult, ssize_t); + if (lg_dirty_mult >= 0) { + malloc_cprintf(write_cb, cbopaque, + "Min active:dirty page ratio: %u:1\n", + (1U << lg_dirty_mult)); + } else { + malloc_cprintf(write_cb, cbopaque, + "Min active:dirty page ratio: N/A\n"); + } CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t); CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t); CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t); diff --git a/test/integration/chunk.c b/test/integration/chunk.c index 89938504..de45bc51 100644 --- a/test/integration/chunk.c +++ b/test/integration/chunk.c @@ -2,13 +2,8 @@ chunk_alloc_t *old_alloc; chunk_dalloc_t *old_dalloc; - -bool -chunk_dalloc(void *chunk, size_t size, unsigned arena_ind) -{ - - return (old_dalloc(chunk, size, arena_ind)); -} +chunk_purge_t *old_purge; +bool purged; void * chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, @@ -18,36 +13,79 @@ chunk_alloc(void *new_addr, size_t size, size_t alignment, bool *zero, return (old_alloc(new_addr, size, alignment, zero, arena_ind)); } +bool +chunk_dalloc(void *chunk, size_t size, unsigned arena_ind) +{ + + return (old_dalloc(chunk, size, arena_ind)); +} + +bool +chunk_purge(void *chunk, size_t offset, size_t length, unsigned arena_ind) +{ + + purged = true; + return (old_purge(chunk, offset, length, arena_ind)); +} + TEST_BEGIN(test_chunk) { void *p; chunk_alloc_t *new_alloc; chunk_dalloc_t *new_dalloc; - size_t old_size, new_size; + chunk_purge_t *new_purge; + size_t old_size, new_size, huge0, huge1, huge2, sz; new_alloc = chunk_alloc; new_dalloc = chunk_dalloc; + new_purge = chunk_purge; old_size = sizeof(chunk_alloc_t *); new_size = sizeof(chunk_alloc_t *); - assert_d_eq(mallctl("arena.0.chunk.alloc", &old_alloc, - &old_size, &new_alloc, new_size), 0, - "Unexpected alloc error"); - assert_ptr_ne(old_alloc, new_alloc, - "Unexpected alloc error"); + assert_d_eq(mallctl("arena.0.chunk.alloc", &old_alloc, &old_size, + &new_alloc, new_size), 0, "Unexpected alloc error"); + assert_ptr_ne(old_alloc, new_alloc, "Unexpected alloc error"); + assert_d_eq(mallctl("arena.0.chunk.dalloc", &old_dalloc, &old_size, &new_dalloc, new_size), 0, "Unexpected dalloc error"); assert_ptr_ne(old_dalloc, new_dalloc, "Unexpected dalloc error"); + assert_d_eq(mallctl("arena.0.chunk.purge", &old_purge, &old_size, + &new_purge, new_size), 0, "Unexpected purge error"); + assert_ptr_ne(old_purge, new_purge, "Unexpected purge error"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("arenas.hchunk.0.size", &huge0, &sz, NULL, 0), 0, + "Unexpected arenas.hchunk.0.size failure"); + assert_d_eq(mallctl("arenas.hchunk.1.size", &huge1, &sz, NULL, 0), 0, + "Unexpected arenas.hchunk.1.size failure"); + assert_d_eq(mallctl("arenas.hchunk.2.size", &huge2, &sz, NULL, 0), 0, + "Unexpected arenas.hchunk.2.size failure"); + if (huge0 * 2 > huge2) { + /* + * There are at least four size classes per doubling, so + * xallocx() from size=huge2 to size=huge1 is guaranteed to + * leave trailing purgeable memory. + */ + p = mallocx(huge2, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + purged = false; + assert_zu_eq(xallocx(p, huge1, 0, 0), huge1, + "Unexpected xallocx() failure"); + assert_true(purged, "Unexpected purge"); + dallocx(p, 0); + } + p = mallocx(42, 0); - assert_ptr_ne(p, NULL, "Unexpected alloc error"); + assert_ptr_not_null(p, "Unexpected mallocx() error"); free(p); - assert_d_eq(mallctl("arena.0.chunk.alloc", NULL, - NULL, &old_alloc, old_size), 0, - "Unexpected alloc error"); + assert_d_eq(mallctl("arena.0.chunk.alloc", NULL, NULL, &old_alloc, + old_size), 0, "Unexpected alloc error"); assert_d_eq(mallctl("arena.0.chunk.dalloc", NULL, NULL, &old_dalloc, old_size), 0, "Unexpected dalloc error"); + assert_d_eq(mallctl("arena.0.chunk.purge", NULL, NULL, &old_purge, + old_size), 0, "Unexpected purge error"); } TEST_END diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 5960496f..31ada191 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -348,6 +348,38 @@ TEST_BEGIN(test_thread_arena) } TEST_END +TEST_BEGIN(test_arena_i_lg_dirty_mult) +{ + ssize_t lg_dirty_mult, orig_lg_dirty_mult, prev_lg_dirty_mult; + size_t sz = sizeof(ssize_t); + + assert_d_eq(mallctl("arena.0.lg_dirty_mult", &orig_lg_dirty_mult, &sz, + NULL, 0), 0, "Unexpected mallctl() failure"); + + lg_dirty_mult = -2; + assert_d_eq(mallctl("arena.0.lg_dirty_mult", NULL, NULL, + &lg_dirty_mult, sizeof(ssize_t)), EFAULT, + "Unexpected mallctl() success"); + + lg_dirty_mult = (sizeof(size_t) << 3); + assert_d_eq(mallctl("arena.0.lg_dirty_mult", NULL, NULL, + &lg_dirty_mult, sizeof(ssize_t)), EFAULT, + "Unexpected mallctl() success"); + + for (prev_lg_dirty_mult = orig_lg_dirty_mult, lg_dirty_mult = -1; + lg_dirty_mult < (sizeof(ssize_t) << 3); prev_lg_dirty_mult = + lg_dirty_mult, lg_dirty_mult++) { + ssize_t old_lg_dirty_mult; + + assert_d_eq(mallctl("arena.0.lg_dirty_mult", &old_lg_dirty_mult, + &sz, &lg_dirty_mult, sizeof(ssize_t)), 0, + "Unexpected mallctl() failure"); + assert_zd_eq(old_lg_dirty_mult, prev_lg_dirty_mult, + "Unexpected old arena.0.lg_dirty_mult"); + } +} +TEST_END + TEST_BEGIN(test_arena_i_purge) { unsigned narenas; @@ -427,6 +459,38 @@ TEST_BEGIN(test_arenas_initialized) } TEST_END +TEST_BEGIN(test_arenas_lg_dirty_mult) +{ + ssize_t lg_dirty_mult, orig_lg_dirty_mult, prev_lg_dirty_mult; + size_t sz = sizeof(ssize_t); + + assert_d_eq(mallctl("arenas.lg_dirty_mult", &orig_lg_dirty_mult, &sz, + NULL, 0), 0, "Unexpected mallctl() failure"); + + lg_dirty_mult = -2; + assert_d_eq(mallctl("arenas.lg_dirty_mult", NULL, NULL, + &lg_dirty_mult, sizeof(ssize_t)), EFAULT, + "Unexpected mallctl() success"); + + lg_dirty_mult = (sizeof(size_t) << 3); + assert_d_eq(mallctl("arenas.lg_dirty_mult", NULL, NULL, + &lg_dirty_mult, sizeof(ssize_t)), EFAULT, + "Unexpected mallctl() success"); + + for (prev_lg_dirty_mult = orig_lg_dirty_mult, lg_dirty_mult = -1; + lg_dirty_mult < (sizeof(ssize_t) << 3); prev_lg_dirty_mult = + lg_dirty_mult, lg_dirty_mult++) { + ssize_t old_lg_dirty_mult; + + assert_d_eq(mallctl("arenas.lg_dirty_mult", &old_lg_dirty_mult, + &sz, &lg_dirty_mult, sizeof(ssize_t)), 0, + "Unexpected mallctl() failure"); + assert_zd_eq(old_lg_dirty_mult, prev_lg_dirty_mult, + "Unexpected old arenas.lg_dirty_mult"); + } +} +TEST_END + TEST_BEGIN(test_arenas_constants) { @@ -554,9 +618,11 @@ main(void) test_tcache_none, test_tcache, test_thread_arena, + test_arena_i_lg_dirty_mult, test_arena_i_purge, test_arena_i_dss, test_arenas_initialized, + test_arenas_lg_dirty_mult, test_arenas_constants, test_arenas_bin_constants, test_arenas_lrun_constants, diff --git a/test/unit/rtree.c b/test/unit/rtree.c index 556c4a87..496e03a4 100644 --- a/test/unit/rtree.c +++ b/test/unit/rtree.c @@ -22,7 +22,7 @@ TEST_BEGIN(test_rtree_get_empty) rtree_t rtree; assert_false(rtree_new(&rtree, i, node_alloc, node_dalloc), "Unexpected rtree_new() failure"); - assert_ptr_eq(rtree_get(&rtree, 0), NULL, + assert_ptr_null(rtree_get(&rtree, 0), "rtree_get() should return NULL for empty tree"); rtree_delete(&rtree); } @@ -75,8 +75,8 @@ TEST_BEGIN(test_rtree_bits) "get key=%#"PRIxPTR, i, j, k, keys[j], keys[k]); } - assert_ptr_eq(rtree_get(&rtree, - (((uintptr_t)1) << (sizeof(uintptr_t)*8-i))), NULL, + assert_ptr_null(rtree_get(&rtree, + (((uintptr_t)1) << (sizeof(uintptr_t)*8-i))), "Only leftmost rtree leaf should be set; " "i=%u, j=%u", i, j); rtree_set(&rtree, keys[j], NULL); @@ -117,11 +117,11 @@ TEST_BEGIN(test_rtree_random) for (j = 0; j < NSET; j++) { rtree_set(&rtree, keys[j], NULL); - assert_ptr_eq(rtree_get(&rtree, keys[j]), NULL, + assert_ptr_null(rtree_get(&rtree, keys[j]), "rtree_get() should return previously set value"); } for (j = 0; j < NSET; j++) { - assert_ptr_eq(rtree_get(&rtree, keys[j]), NULL, + assert_ptr_null(rtree_get(&rtree, keys[j]), "rtree_get() should return previously set value"); } From e0a08a14962c8d6b09fd25ba9f3f6c57d5a4f844 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 18 Mar 2015 21:06:58 -0700 Subject: [PATCH 262/352] Restore --enable-ivsalloc. However, unlike before it was removed do not force --enable-ivsalloc when Darwin zone allocator integration is enabled, since the zone allocator code uses ivsalloc() regardless of whether malloc_usable_size() and sallocx() do. This resolves #211. --- ChangeLog | 2 -- INSTALL | 6 +++++ configure.ac | 23 ++++++++++++++++++- .../jemalloc/internal/jemalloc_internal.h.in | 7 ++++++ .../internal/jemalloc_internal_defs.h.in | 6 +++++ src/jemalloc.c | 4 ++-- 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/ChangeLog b/ChangeLog index ef7dbfdb..a462d025 100644 --- a/ChangeLog +++ b/ChangeLog @@ -133,8 +133,6 @@ found in the git revision history: - Remove the "stats.huge.allocated", "stats.huge.nmalloc", and "stats.huge.ndalloc" mallctls. - Remove the --enable-mremap option. - - Remove the --enable-ivsalloc option, and merge its functionality into - --enable-debug. - Remove the "stats.chunks.current", "stats.chunks.total", and "stats.chunks.high" mallctls. diff --git a/INSTALL b/INSTALL index 517fe021..cd760ca2 100644 --- a/INSTALL +++ b/INSTALL @@ -92,6 +92,7 @@ any of the following arguments (not a definitive list) to 'configure': --enable-debug Enable assertions and validation code. This incurs a substantial performance hit, but is very useful during application development. + Implies --enable-ivsalloc. --enable-code-coverage Enable code coverage support, for use during jemalloc test development. @@ -110,6 +111,11 @@ any of the following arguments (not a definitive list) to 'configure': Disable statistics gathering functionality. See the "opt.stats_print" option documentation for usage details. +--enable-ivsalloc + Enable validation code, which verifies that pointers reside within + jemalloc-owned chunks before dereferencing them. This incurs a minor + performance hit. + --enable-prof Enable heap profiling and leak detection functionality. See the "opt.prof" option documentation for usage details. When enabled, there are several diff --git a/configure.ac b/configure.ac index 4ac7ac82..be49743d 100644 --- a/configure.ac +++ b/configure.ac @@ -625,7 +625,8 @@ fi dnl Do not compile with debugging by default. AC_ARG_ENABLE([debug], - [AS_HELP_STRING([--enable-debug], [Build debugging code])], + [AS_HELP_STRING([--enable-debug], + [Build debugging code (implies --enable-ivsalloc)])], [if test "x$enable_debug" = "xno" ; then enable_debug="0" else @@ -637,8 +638,28 @@ fi if test "x$enable_debug" = "x1" ; then AC_DEFINE([JEMALLOC_DEBUG], [ ]) fi +if test "x$enable_debug" = "x1" ; then + AC_DEFINE([JEMALLOC_DEBUG], [ ]) + enable_ivsalloc="1" +fi AC_SUBST([enable_debug]) +dnl Do not validate pointers by default. +AC_ARG_ENABLE([ivsalloc], + [AS_HELP_STRING([--enable-ivsalloc], + [Validate pointers passed through the public API])], +[if test "x$enable_ivsalloc" = "xno" ; then + enable_ivsalloc="0" +else + enable_ivsalloc="1" +fi +], +[enable_ivsalloc="0"] +) +if test "x$enable_ivsalloc" = "x1" ; then + AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) +fi + dnl Only optimize if not debugging. if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS. diff --git a/include/jemalloc/internal/jemalloc_internal.h.in b/include/jemalloc/internal/jemalloc_internal.h.in index 8ed69ce2..b398f31e 100644 --- a/include/jemalloc/internal/jemalloc_internal.h.in +++ b/include/jemalloc/internal/jemalloc_internal.h.in @@ -119,6 +119,13 @@ static const bool config_xmalloc = false #endif ; +static const bool config_ivsalloc = +#ifdef JEMALLOC_IVSALLOC + true +#else + false +#endif + ; #ifdef JEMALLOC_C11ATOMICS #include diff --git a/include/jemalloc/internal/jemalloc_internal_defs.h.in b/include/jemalloc/internal/jemalloc_internal_defs.h.in index 191abc52..a943d23c 100644 --- a/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -186,6 +186,12 @@ #undef JEMALLOC_INTERNAL_FFSL #undef JEMALLOC_INTERNAL_FFS +/* + * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside + * within jemalloc-owned chunks before dereferencing them. + */ +#undef JEMALLOC_IVSALLOC + /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ diff --git a/src/jemalloc.c b/src/jemalloc.c index d5110092..7e9f4860 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -2306,7 +2306,7 @@ je_sallocx(const void *ptr, int flags) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); - if (config_debug) + if (config_ivsalloc) usize = ivsalloc(ptr, config_prof); else usize = isalloc(ptr, config_prof); @@ -2434,7 +2434,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) assert(malloc_initialized() || IS_INITIALIZER); malloc_thread_init(); - if (config_debug) + if (config_ivsalloc) ret = ivsalloc(ptr, config_prof); else ret = (ptr == NULL) ? 0 : isalloc(ptr, config_prof); From 7e336e7359ec50f06ec73f29033c7807148bf476 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 20 Mar 2015 18:08:10 -0700 Subject: [PATCH 263/352] Fix lg_dirty_mult-related stats printing. This regression was introduced by 8d6a3e8321a7767cb2ca0930b85d5d488a8cc659 (Implement dynamic per arena control over dirty page purging.). This resolves #215. --- src/stats.c | 148 +++++++++++++++++++++++++++++----------------------- 1 file changed, 82 insertions(+), 66 deletions(-) diff --git a/src/stats.c b/src/stats.c index f246c8bc..ae74737c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -6,31 +6,31 @@ xmallctl(n, v, &sz, NULL, 0); \ } while (0) -#define CTL_I_GET(n, v, t) do { \ +#define CTL_M1_GET(n, i, v, t) do { \ size_t mib[6]; \ size_t miblen = sizeof(mib) / sizeof(size_t); \ size_t sz = sizeof(t); \ xmallctlnametomib(n, mib, &miblen); \ - mib[2] = i; \ + mib[1] = (i); \ xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ } while (0) -#define CTL_J_GET(n, v, t) do { \ +#define CTL_M2_GET(n, i, v, t) do { \ size_t mib[6]; \ size_t miblen = sizeof(mib) / sizeof(size_t); \ size_t sz = sizeof(t); \ xmallctlnametomib(n, mib, &miblen); \ - mib[2] = j; \ + mib[2] = (i); \ xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ } while (0) -#define CTL_IJ_GET(n, v, t) do { \ +#define CTL_M2_M4_GET(n, i, j, v, t) do { \ size_t mib[6]; \ size_t miblen = sizeof(mib) / sizeof(size_t); \ size_t sz = sizeof(t); \ xmallctlnametomib(n, mib, &miblen); \ - mib[2] = i; \ - mib[4] = j; \ + mib[2] = (i); \ + mib[4] = (j); \ xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ } while (0) @@ -82,7 +82,8 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, for (j = 0, in_gap = false; j < nbins; j++) { uint64_t nruns; - CTL_IJ_GET("stats.arenas.0.bins.0.nruns", &nruns, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nruns", i, j, &nruns, + uint64_t); if (nruns == 0) in_gap = true; else { @@ -98,27 +99,28 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, " ---\n"); in_gap = false; } - CTL_J_GET("arenas.bin.0.size", ®_size, size_t); - CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t); - CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc", + CTL_M2_GET("arenas.bin.0.size", j, ®_size, size_t); + CTL_M2_GET("arenas.bin.0.nregs", j, &nregs, uint32_t); + CTL_M2_GET("arenas.bin.0.run_size", j, &run_size, + size_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nmalloc", i, j, &nmalloc, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc", + CTL_M2_M4_GET("stats.arenas.0.bins.0.ndalloc", i, j, &ndalloc, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.curregs", + CTL_M2_M4_GET("stats.arenas.0.bins.0.curregs", i, j, &curregs, size_t); - CTL_IJ_GET("stats.arenas.0.bins.0.nrequests", + CTL_M2_M4_GET("stats.arenas.0.bins.0.nrequests", i, j, &nrequests, uint64_t); if (config_tcache) { - CTL_IJ_GET("stats.arenas.0.bins.0.nfills", - &nfills, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.nflushes", - &nflushes, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nfills", i, + j, &nfills, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nflushes", + i, j, &nflushes, uint64_t); } - CTL_IJ_GET("stats.arenas.0.bins.0.nreruns", &reruns, - uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns, - size_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.nreruns", i, j, + &reruns, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.bins.0.curruns", i, j, + &curruns, size_t); availregs = nregs * curruns; milli = (availregs != 0) ? (1000 * curregs) / availregs @@ -179,18 +181,18 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, uint64_t nmalloc, ndalloc, nrequests; size_t run_size, curruns; - CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc, + CTL_M2_M4_GET("stats.arenas.0.lruns.0.nmalloc", i, j, &nmalloc, uint64_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.ndalloc", &ndalloc, - uint64_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.nrequests", &nrequests, + CTL_M2_M4_GET("stats.arenas.0.lruns.0.ndalloc", i, j, &ndalloc, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.lruns.0.nrequests", i, j, + &nrequests, uint64_t); if (nrequests == 0) in_gap = true; else { - CTL_J_GET("arenas.lrun.0.size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns, - size_t); + CTL_M2_GET("arenas.lrun.0.size", j, &run_size, size_t); + CTL_M2_M4_GET("stats.arenas.0.lruns.0.curruns", i, j, + &curruns, size_t); if (in_gap) { malloc_cprintf(write_cb, cbopaque, " ---\n"); @@ -226,19 +228,19 @@ stats_arena_hchunks_print(void (*write_cb)(void *, const char *), uint64_t nmalloc, ndalloc, nrequests; size_t hchunk_size, curhchunks; - CTL_IJ_GET("stats.arenas.0.hchunks.0.nmalloc", &nmalloc, - uint64_t); - CTL_IJ_GET("stats.arenas.0.hchunks.0.ndalloc", &ndalloc, - uint64_t); - CTL_IJ_GET("stats.arenas.0.hchunks.0.nrequests", &nrequests, - uint64_t); + CTL_M2_M4_GET("stats.arenas.0.hchunks.0.nmalloc", i, j, + &nmalloc, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.hchunks.0.ndalloc", i, j, + &ndalloc, uint64_t); + CTL_M2_M4_GET("stats.arenas.0.hchunks.0.nrequests", i, j, + &nrequests, uint64_t); if (nrequests == 0) in_gap = true; else { - CTL_J_GET("arenas.hchunk.0.size", &hchunk_size, + CTL_M2_GET("arenas.hchunk.0.size", j, &hchunk_size, size_t); - CTL_IJ_GET("stats.arenas.0.hchunks.0.curhchunks", - &curhchunks, size_t); + CTL_M2_M4_GET("stats.arenas.0.hchunks.0.curhchunks", i, + j, &curhchunks, size_t); if (in_gap) { malloc_cprintf(write_cb, cbopaque, " ---\n"); @@ -277,26 +279,26 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.page", &page, size_t); - CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned); + CTL_M2_GET("stats.arenas.0.nthreads", i, &nthreads, unsigned); malloc_cprintf(write_cb, cbopaque, "assigned threads: %u\n", nthreads); - CTL_I_GET("stats.arenas.0.dss", &dss, const char *); + CTL_M2_GET("stats.arenas.0.dss", i, &dss, const char *); malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n", dss); - CTL_I_GET("stats.arenas.0.lg_dirty_mult", &lg_dirty_mult, ssize_t); + CTL_M1_GET("arena.0.lg_dirty_mult", i, &lg_dirty_mult, ssize_t); if (lg_dirty_mult >= 0) { malloc_cprintf(write_cb, cbopaque, - "Min active:dirty page ratio: %u:1\n", + "min active:dirty page ratio: %u:1\n", (1U << lg_dirty_mult)); } else { malloc_cprintf(write_cb, cbopaque, - "Min active:dirty page ratio: N/A\n"); + "min active:dirty page ratio: N/A\n"); } - CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t); - CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t); - CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t); - CTL_I_GET("stats.arenas.0.nmadvise", &nmadvise, uint64_t); - CTL_I_GET("stats.arenas.0.purged", &purged, uint64_t); + CTL_M2_GET("stats.arenas.0.pactive", i, &pactive, size_t); + CTL_M2_GET("stats.arenas.0.pdirty", i, &pdirty, size_t); + CTL_M2_GET("stats.arenas.0.npurge", i, &npurge, uint64_t); + CTL_M2_GET("stats.arenas.0.nmadvise", i, &nmadvise, uint64_t); + CTL_M2_GET("stats.arenas.0.purged", i, &purged, uint64_t); malloc_cprintf(write_cb, cbopaque, "dirty pages: %zu:%zu active:dirty, %"PRIu64" sweep%s," " %"PRIu64" madvise%s, %"PRIu64" purged\n", @@ -306,26 +308,31 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, " allocated nmalloc ndalloc" " nrequests\n"); - CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t); - CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t); - CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t); - CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t); + CTL_M2_GET("stats.arenas.0.small.allocated", i, &small_allocated, + size_t); + CTL_M2_GET("stats.arenas.0.small.nmalloc", i, &small_nmalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.small.ndalloc", i, &small_ndalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.small.nrequests", i, &small_nrequests, + uint64_t); malloc_cprintf(write_cb, cbopaque, "small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 "\n", small_allocated, small_nmalloc, small_ndalloc, small_nrequests); - CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t); - CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t); - CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t); - CTL_I_GET("stats.arenas.0.large.nrequests", &large_nrequests, uint64_t); + CTL_M2_GET("stats.arenas.0.large.allocated", i, &large_allocated, + size_t); + CTL_M2_GET("stats.arenas.0.large.nmalloc", i, &large_nmalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.large.ndalloc", i, &large_ndalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.large.nrequests", i, &large_nrequests, + uint64_t); malloc_cprintf(write_cb, cbopaque, "large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 "\n", large_allocated, large_nmalloc, large_ndalloc, large_nrequests); - CTL_I_GET("stats.arenas.0.huge.allocated", &huge_allocated, size_t); - CTL_I_GET("stats.arenas.0.huge.nmalloc", &huge_nmalloc, uint64_t); - CTL_I_GET("stats.arenas.0.huge.ndalloc", &huge_ndalloc, uint64_t); - CTL_I_GET("stats.arenas.0.huge.nrequests", &huge_nrequests, uint64_t); + CTL_M2_GET("stats.arenas.0.huge.allocated", i, &huge_allocated, size_t); + CTL_M2_GET("stats.arenas.0.huge.nmalloc", i, &huge_nmalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.huge.ndalloc", i, &huge_ndalloc, uint64_t); + CTL_M2_GET("stats.arenas.0.huge.nrequests", i, &huge_nrequests, + uint64_t); malloc_cprintf(write_cb, cbopaque, "huge: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64 "\n", @@ -339,11 +346,12 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, small_nrequests + large_nrequests + huge_nrequests); malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", pactive * page); - CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); + CTL_M2_GET("stats.arenas.0.mapped", i, &mapped, size_t); malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); - CTL_I_GET("stats.arenas.0.metadata.mapped", &metadata_mapped, size_t); - CTL_I_GET("stats.arenas.0.metadata.allocated", &metadata_allocated, + CTL_M2_GET("stats.arenas.0.metadata.mapped", i, &metadata_mapped, + size_t); + CTL_M2_GET("stats.arenas.0.metadata.allocated", i, &metadata_allocated, size_t); malloc_cprintf(write_cb, cbopaque, "metadata: mapped: %zu, allocated: %zu\n", metadata_mapped, @@ -464,6 +472,14 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %zd\n", ssv); \ } +#define OPT_WRITE_SSIZE_T_MUTABLE(n, m) \ + ssize_t ssv2; \ + if (je_mallctl("opt."#n, &ssv, &sssz, NULL, 0) == 0 && \ + je_mallctl(#m, &ssv2, &sssz, NULL, 0) == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %zd ("#m": %zd)\n", ssv, \ + ssv2); \ + } #define OPT_WRITE_CHAR_P(n) \ if (je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0) == 0) { \ malloc_cprintf(write_cb, cbopaque, \ @@ -476,7 +492,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, OPT_WRITE_SIZE_T(lg_chunk) OPT_WRITE_CHAR_P(dss) OPT_WRITE_SIZE_T(narenas) - OPT_WRITE_SSIZE_T(lg_dirty_mult) + OPT_WRITE_SSIZE_T_MUTABLE(lg_dirty_mult, arenas.lg_dirty_mult) OPT_WRITE_BOOL(stats_print) OPT_WRITE_CHAR_P(junk) OPT_WRITE_SIZE_T(quarantine) @@ -519,7 +535,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.page", &sv, size_t); malloc_cprintf(write_cb, cbopaque, "Page size: %zu\n", sv); - CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t); + CTL_GET("arenas.lg_dirty_mult", &ssv, ssize_t); if (ssv >= 0) { malloc_cprintf(write_cb, cbopaque, "Min active:dirty page ratio per arena: %u:1\n", From fd5901ce3083cd3277b87aa414884d7628e2d509 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Sat, 21 Mar 2015 10:18:39 -0700 Subject: [PATCH 264/352] Fix a compile error caused by mixed declarations and code. --- src/stats.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/stats.c b/src/stats.c index ae74737c..b41b458b 100644 --- a/src/stats.c +++ b/src/stats.c @@ -472,14 +472,15 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %zd\n", ssv); \ } -#define OPT_WRITE_SSIZE_T_MUTABLE(n, m) \ +#define OPT_WRITE_SSIZE_T_MUTABLE(n, m) { \ ssize_t ssv2; \ if (je_mallctl("opt."#n, &ssv, &sssz, NULL, 0) == 0 && \ je_mallctl(#m, &ssv2, &sssz, NULL, 0) == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %zd ("#m": %zd)\n", ssv, \ ssv2); \ - } + } \ +} #define OPT_WRITE_CHAR_P(n) \ if (je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0) == 0) { \ malloc_cprintf(write_cb, cbopaque, \ From 8ad6bf360f9ca5c6c9a1d8e5825ee473bb4697da Mon Sep 17 00:00:00 2001 From: Igor Podlesny Date: Sun, 22 Mar 2015 01:30:02 +0700 Subject: [PATCH 265/352] Fix indentation inconsistencies. --- include/jemalloc/internal/util.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/jemalloc/internal/util.h b/include/jemalloc/internal/util.h index 5ad4933d..001cd092 100644 --- a/include/jemalloc/internal/util.h +++ b/include/jemalloc/internal/util.h @@ -22,17 +22,17 @@ * uninitialized. */ #ifdef JEMALLOC_CC_SILENCE -# define JEMALLOC_CC_SILENCE_INIT(v) = v +# define JEMALLOC_CC_SILENCE_INIT(v) = v #else -# define JEMALLOC_CC_SILENCE_INIT(v) +# define JEMALLOC_CC_SILENCE_INIT(v) #endif #ifdef __GNUC__ -#define likely(x) __builtin_expect(!!(x), 1) -#define unlikely(x) __builtin_expect(!!(x), 0) +# define likely(x) __builtin_expect(!!(x), 1) +# define unlikely(x) __builtin_expect(!!(x), 0) #else -#define likely(x) !!(x) -#define unlikely(x) !!(x) +# define likely(x) !!(x) +# define unlikely(x) !!(x) #endif /* From 4acd75a694173186e9e0399d2855f05ce8553008 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Mon, 23 Mar 2015 17:25:57 -0700 Subject: [PATCH 266/352] Add the "stats.allocated" mallctl. --- ChangeLog | 2 ++ doc/jemalloc.xml.in | 23 +++++++++++++-- include/jemalloc/internal/base.h | 2 +- include/jemalloc/internal/ctl.h | 1 + include/jemalloc/internal/private_symbols.txt | 2 +- src/base.c | 29 ++++++++++++++----- src/ctl.c | 23 ++++++++++----- src/stats.c | 8 +++-- 8 files changed, 67 insertions(+), 23 deletions(-) diff --git a/ChangeLog b/ChangeLog index a462d025..26075766 100644 --- a/ChangeLog +++ b/ChangeLog @@ -63,6 +63,8 @@ found in the git revision history: - Add metadata statistics, which are accessible via the "stats.metadata", "stats.arenas..metadata.mapped", and "stats.arenas..metadata.allocated" mallctls. + - Add the "stats.resident" mallctl, which reports the upper limit of + physically resident memory mapped by the allocator. - Add the "prof.gdump" mallctl, which makes it possible to toggle the gdump feature on/off during program execution. - Add sdallocx(), which implements sized deallocation. The primary diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index 01ac38c3..adff6a4d 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1938,6 +1938,23 @@ malloc_conf = "xmalloc:true";]]> linkend="stats.arenas.i.metadata.allocated">stats.arenas.<i>.metadata.allocated). + + + stats.resident + (size_t) + r- + [] + + Maximum number of bytes in physically resident data + pages mapped by the allocator, comprising all pages dedicated to + allocator metadata, pages backing active allocations, and unused dirty + pages. This is a maximum rather than precise because pages may not + actually be physically resident if they correspond to demand-zeroed + virtual memory that has not yet been touched. This is a multiple of the + page size, and is larger than stats.active. + + stats.mapped @@ -1945,10 +1962,10 @@ malloc_conf = "xmalloc:true";]]> r- [] - Total number of bytes in chunks mapped on behalf of the - application. This is a multiple of the chunk size, and is at least as + Total number of bytes in active chunks mapped by the + allocator. This is a multiple of the chunk size, and is at least as large as stats.active. This + linkend="stats.resident">stats.resident. This does not include inactive chunks. diff --git a/include/jemalloc/internal/base.h b/include/jemalloc/internal/base.h index bec76b32..39e46ee4 100644 --- a/include/jemalloc/internal/base.h +++ b/include/jemalloc/internal/base.h @@ -10,7 +10,7 @@ #ifdef JEMALLOC_H_EXTERNS void *base_alloc(size_t size); -size_t base_allocated_get(void); +void base_stats_get(size_t *allocated, size_t *resident, size_t *mapped); bool base_boot(void); void base_prefork(void); void base_postfork_parent(void); diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index ab9c9862..7c2a4bea 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -53,6 +53,7 @@ struct ctl_stats_s { size_t allocated; size_t active; size_t metadata; + size_t resident; size_t mapped; unsigned narenas; ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt index bc0f2a6a..aaf69786 100644 --- a/include/jemalloc/internal/private_symbols.txt +++ b/include/jemalloc/internal/private_symbols.txt @@ -111,11 +111,11 @@ atomic_sub_uint32 atomic_sub_uint64 atomic_sub_z base_alloc -base_allocated_get base_boot base_postfork_child base_postfork_parent base_prefork +base_stats_get bitmap_full bitmap_get bitmap_info_init diff --git a/src/base.c b/src/base.c index 01c62df4..1a9b829a 100644 --- a/src/base.c +++ b/src/base.c @@ -8,6 +8,8 @@ static malloc_mutex_t base_mtx; static extent_tree_t base_avail_szad; static extent_node_t *base_nodes; static size_t base_allocated; +static size_t base_resident; +static size_t base_mapped; /******************************************************************************/ @@ -54,11 +56,15 @@ base_chunk_alloc(size_t minsize) base_node_dalloc(node); return (NULL); } + base_mapped += csize; if (node == NULL) { + node = (extent_node_t *)addr; + addr = (void *)((uintptr_t)addr + nsize); csize -= nsize; - node = (extent_node_t *)((uintptr_t)addr + csize); - if (config_stats) + if (config_stats) { base_allocated += nsize; + base_resident += PAGE_CEILING(nsize); + } } extent_node_init(node, NULL, addr, csize, true); return (node); @@ -106,23 +112,30 @@ base_alloc(size_t size) extent_tree_szad_insert(&base_avail_szad, node); } else base_node_dalloc(node); - if (config_stats) + if (config_stats) { base_allocated += csize; + /* + * Add one PAGE to base_resident for every page boundary that is + * crossed by the new allocation. + */ + base_resident += PAGE_CEILING((uintptr_t)ret + csize) - + PAGE_CEILING((uintptr_t)ret); + } JEMALLOC_VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); label_return: malloc_mutex_unlock(&base_mtx); return (ret); } -size_t -base_allocated_get(void) +void +base_stats_get(size_t *allocated, size_t *resident, size_t *mapped) { - size_t ret; malloc_mutex_lock(&base_mtx); - ret = base_allocated; + *allocated = base_allocated; + *resident = base_resident; + *mapped = base_mapped; malloc_mutex_unlock(&base_mtx); - return (ret); } bool diff --git a/src/ctl.c b/src/ctl.c index 447b8776..0ed8ddd4 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -194,6 +194,7 @@ CTL_PROTO(stats_cactive) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) CTL_PROTO(stats_metadata) +CTL_PROTO(stats_resident) CTL_PROTO(stats_mapped) /******************************************************************************/ @@ -469,6 +470,7 @@ static const ctl_named_node_t stats_node[] = { {NAME("allocated"), CTL(stats_allocated)}, {NAME("active"), CTL(stats_active)}, {NAME("metadata"), CTL(stats_metadata)}, + {NAME("resident"), CTL(stats_resident)}, {NAME("mapped"), CTL(stats_mapped)}, {NAME("arenas"), CHILD(indexed, stats_arenas)} }; @@ -711,17 +713,23 @@ ctl_refresh(void) } if (config_stats) { + size_t base_allocated, base_resident, base_mapped; + base_stats_get(&base_allocated, &base_resident, &base_mapped); ctl_stats.allocated = - ctl_stats.arenas[ctl_stats.narenas].allocated_small - + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large - + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge; + ctl_stats.arenas[ctl_stats.narenas].allocated_small + + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large + + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_huge; ctl_stats.active = (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE); - ctl_stats.metadata = base_allocated_get() - + ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped - + ctl_stats.arenas[ctl_stats.narenas].astats + ctl_stats.metadata = base_allocated + + ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped + + ctl_stats.arenas[ctl_stats.narenas].astats .metadata_allocated; - ctl_stats.mapped = + ctl_stats.resident = base_resident + + ctl_stats.arenas[ctl_stats.narenas].astats.metadata_mapped + + ((ctl_stats.arenas[ctl_stats.narenas].pactive + + ctl_stats.arenas[ctl_stats.narenas].pdirty) << LG_PAGE); + ctl_stats.mapped = base_mapped + ctl_stats.arenas[ctl_stats.narenas].astats.mapped; } @@ -1976,6 +1984,7 @@ CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats.metadata, size_t) +CTL_RO_CGEN(config_stats, stats_resident, ctl_stats.resident, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) diff --git a/src/stats.c b/src/stats.c index b41b458b..c5cea5e6 100644 --- a/src/stats.c +++ b/src/stats.c @@ -573,16 +573,18 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, if (config_stats) { size_t *cactive; - size_t allocated, active, metadata, mapped; + size_t allocated, active, metadata, resident, mapped; CTL_GET("stats.cactive", &cactive, size_t *); CTL_GET("stats.allocated", &allocated, size_t); CTL_GET("stats.active", &active, size_t); CTL_GET("stats.metadata", &metadata, size_t); + CTL_GET("stats.resident", &resident, size_t); CTL_GET("stats.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, - "Allocated: %zu, active: %zu, metadata: %zu, mapped: %zu\n", - allocated, active, metadata, mapped); + "Allocated: %zu, active: %zu, metadata: %zu, resident: %zu," + " mapped: %zu\n", allocated, active, metadata, resident, + mapped); malloc_cprintf(write_cb, cbopaque, "Current active ceiling: %zu\n", atomic_read_z(cactive)); From ef0a0cc3283ea561a40b33f4325d54bbc351de21 Mon Sep 17 00:00:00 2001 From: Igor Podlesny Date: Sun, 22 Mar 2015 23:49:58 +0700 Subject: [PATCH 267/352] We have pages_unmap(ret, size) so we use it. --- src/chunk_mmap.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/chunk_mmap.c b/src/chunk_mmap.c index 7e02c102..30ac10be 100644 --- a/src/chunk_mmap.c +++ b/src/chunk_mmap.c @@ -40,15 +40,7 @@ pages_map(void *addr, size_t size) /* * We succeeded in mapping memory, but not in the right place. */ - if (munmap(ret, size) == -1) { - char buf[BUFERROR_BUF]; - - buferror(get_errno(), buf, sizeof(buf)); - malloc_printf(" Date: Tue, 24 Mar 2015 12:33:12 -0700 Subject: [PATCH 268/352] Fix arena_get() usage. Fix arena_get() calls that specify refresh_if_missing=false. In ctl_refresh() and ctl.c's arena_purge(), these calls attempted to only refresh once, but did so in an unreliable way. arena_i_lg_dirty_mult_ctl() was simply wrong to pass refresh_if_missing=false. --- src/ctl.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/ctl.c b/src/ctl.c index 0ed8ddd4..44935467 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -684,6 +684,7 @@ ctl_refresh(void) { tsd_t *tsd; unsigned i; + bool refreshed; VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); /* @@ -694,8 +695,13 @@ ctl_refresh(void) ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]); tsd = tsd_fetch(); - for (i = 0; i < ctl_stats.narenas; i++) - tarenas[i] = arena_get(tsd, i, false, (i == 0)); + for (i = 0, refreshed = false; i < ctl_stats.narenas; i++) { + tarenas[i] = arena_get(tsd, i, false, false); + if (tarenas[i] == NULL && !refreshed) { + tarenas[i] = arena_get(tsd, i, false, true); + refreshed = true; + } + } for (i = 0; i < ctl_stats.narenas; i++) { if (tarenas[i] != NULL) @@ -1538,11 +1544,17 @@ arena_purge(unsigned arena_ind) { tsd_t *tsd; unsigned i; + bool refreshed; VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); tsd = tsd_fetch(); - for (i = 0; i < ctl_stats.narenas; i++) - tarenas[i] = arena_get(tsd, i, false, (i == 0)); + for (i = 0, refreshed = false; i < ctl_stats.narenas; i++) { + tarenas[i] = arena_get(tsd, i, false, false); + if (tarenas[i] == NULL && !refreshed) { + tarenas[i] = arena_get(tsd, i, false, true); + refreshed = true; + } + } if (arena_ind == ctl_stats.narenas) { unsigned i; @@ -1638,7 +1650,7 @@ arena_i_lg_dirty_mult_ctl(const size_t *mib, size_t miblen, void *oldp, unsigned arena_ind = mib[1]; arena_t *arena; - arena = arena_get(tsd_fetch(), arena_ind, false, (arena_ind == 0)); + arena = arena_get(tsd_fetch(), arena_ind, false, true); if (arena == NULL) { ret = EFAULT; goto label_return; From bd16ea49c3e36706a52ef9c8f560813c167fa085 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 24 Mar 2015 15:59:28 -0700 Subject: [PATCH 269/352] Fix signed/unsigned comparison in arena_lg_dirty_mult_valid(). --- src/arena.c | 3 ++- test/unit/mallctl.c | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/arena.c b/src/arena.c index 7272682d..d38ffc6b 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1037,7 +1037,8 @@ static bool arena_lg_dirty_mult_valid(ssize_t lg_dirty_mult) { - return (lg_dirty_mult >= -1 && lg_dirty_mult < (sizeof(size_t) << 3)); + return (lg_dirty_mult >= -1 && lg_dirty_mult < (ssize_t)(sizeof(size_t) + << 3)); } ssize_t diff --git a/test/unit/mallctl.c b/test/unit/mallctl.c index 31ada191..29823a6c 100644 --- a/test/unit/mallctl.c +++ b/test/unit/mallctl.c @@ -367,8 +367,8 @@ TEST_BEGIN(test_arena_i_lg_dirty_mult) "Unexpected mallctl() success"); for (prev_lg_dirty_mult = orig_lg_dirty_mult, lg_dirty_mult = -1; - lg_dirty_mult < (sizeof(ssize_t) << 3); prev_lg_dirty_mult = - lg_dirty_mult, lg_dirty_mult++) { + lg_dirty_mult < (ssize_t)(sizeof(size_t) << 3); prev_lg_dirty_mult + = lg_dirty_mult, lg_dirty_mult++) { ssize_t old_lg_dirty_mult; assert_d_eq(mallctl("arena.0.lg_dirty_mult", &old_lg_dirty_mult, @@ -478,7 +478,7 @@ TEST_BEGIN(test_arenas_lg_dirty_mult) "Unexpected mallctl() success"); for (prev_lg_dirty_mult = orig_lg_dirty_mult, lg_dirty_mult = -1; - lg_dirty_mult < (sizeof(ssize_t) << 3); prev_lg_dirty_mult = + lg_dirty_mult < (ssize_t)(sizeof(size_t) << 3); prev_lg_dirty_mult = lg_dirty_mult, lg_dirty_mult++) { ssize_t old_lg_dirty_mult; From 562d266511053a51406e91c78eba640cb46ad9c8 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Tue, 24 Mar 2015 16:36:12 -0700 Subject: [PATCH 270/352] Add the "stats.arenas..lg_dirty_mult" mallctl. --- ChangeLog | 6 +++++- doc/jemalloc.xml.in | 12 ++++++++++++ include/jemalloc/internal/arena.h | 5 +++-- include/jemalloc/internal/ctl.h | 1 + src/arena.c | 8 +++++--- src/ctl.c | 11 ++++++++--- src/stats.c | 11 +---------- 7 files changed, 35 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 26075766..8cc214ab 100644 --- a/ChangeLog +++ b/ChangeLog @@ -38,7 +38,8 @@ found in the git revision history: "opt.prof_thread_active_init", "prof.thread_active_init", and "thread.prof.active" mallctls. - Add support for per arena application-specified chunk allocators, configured - via the "arena.chunk.alloc" and "arena.chunk.dalloc" mallctls. + via the "arena.chunk.alloc", "arena.chunk.dalloc", and + "arena..chunk.purge" mallctls. - Refactor huge allocation to be managed by arenas, so that arenas now function as general purpose independent allocators. This is important in the context of user-specified chunk allocators, aside from the scalability @@ -65,6 +66,9 @@ found in the git revision history: "stats.arenas..metadata.allocated" mallctls. - Add the "stats.resident" mallctl, which reports the upper limit of physically resident memory mapped by the allocator. + - Add per arena control over unused dirty page purging, via the + "arenas.lg_dirty_mult", "arena..lg_dirty_mult", and + "stats.arenas..lg_dirty_mult" mallctls. - Add the "prof.gdump" mallctl, which makes it possible to toggle the gdump feature on/off during program execution. - Add sdallocx(), which implements sized deallocation. The primary diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index adff6a4d..d3f36164 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1983,6 +1983,18 @@ malloc_conf = "xmalloc:true";]]> + + + stats.arenas.<i>.lg_dirty_mult + (ssize_t) + r- + + Minimum ratio (log base 2) of active to dirty pages. + See opt.lg_dirty_mult + for details. + + stats.arenas.<i>.nthreads diff --git a/include/jemalloc/internal/arena.h b/include/jemalloc/internal/arena.h index 56ee74aa..dff99fb4 100644 --- a/include/jemalloc/internal/arena.h +++ b/include/jemalloc/internal/arena.h @@ -470,8 +470,9 @@ dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); ssize_t arena_lg_dirty_mult_default_get(void); bool arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult); -void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, - size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, +void arena_stats_merge(arena_t *arena, const char **dss, + ssize_t *lg_dirty_mult, size_t *nactive, size_t *ndirty, + arena_stats_t *astats, malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats); arena_t *arena_new(unsigned ind); void arena_boot(void); diff --git a/include/jemalloc/internal/ctl.h b/include/jemalloc/internal/ctl.h index 7c2a4bea..751c14b5 100644 --- a/include/jemalloc/internal/ctl.h +++ b/include/jemalloc/internal/ctl.h @@ -34,6 +34,7 @@ struct ctl_arena_stats_s { bool initialized; unsigned nthreads; const char *dss; + ssize_t lg_dirty_mult; size_t pactive; size_t pdirty; arena_stats_t astats; diff --git a/src/arena.c b/src/arena.c index d38ffc6b..bc13d209 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2657,14 +2657,16 @@ arena_lg_dirty_mult_default_set(ssize_t lg_dirty_mult) } void -arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, - size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats, malloc_huge_stats_t *hstats) +arena_stats_merge(arena_t *arena, const char **dss, ssize_t *lg_dirty_mult, + size_t *nactive, size_t *ndirty, arena_stats_t *astats, + malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats, + malloc_huge_stats_t *hstats) { unsigned i; malloc_mutex_lock(&arena->lock); *dss = dss_prec_names[arena->dss_prec]; + *lg_dirty_mult = arena->lg_dirty_mult; *nactive += arena->nactive; *ndirty += arena->ndirty; diff --git a/src/ctl.c b/src/ctl.c index 44935467..d215b19b 100644 --- a/src/ctl.c +++ b/src/ctl.c @@ -181,6 +181,7 @@ CTL_PROTO(stats_arenas_i_hchunks_j_curhchunks) INDEX_PROTO(stats_arenas_i_hchunks_j) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_dss) +CTL_PROTO(stats_arenas_i_lg_dirty_mult) CTL_PROTO(stats_arenas_i_pactive) CTL_PROTO(stats_arenas_i_pdirty) CTL_PROTO(stats_arenas_i_mapped) @@ -443,6 +444,7 @@ static const ctl_indexed_node_t stats_arenas_i_hchunks_node[] = { static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, {NAME("dss"), CTL(stats_arenas_i_dss)}, + {NAME("lg_dirty_mult"), CTL(stats_arenas_i_lg_dirty_mult)}, {NAME("pactive"), CTL(stats_arenas_i_pactive)}, {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, @@ -524,6 +526,7 @@ ctl_arena_clear(ctl_arena_stats_t *astats) { astats->dss = dss_prec_names[dss_prec_limit]; + astats->lg_dirty_mult = -1; astats->pactive = 0; astats->pdirty = 0; if (config_stats) { @@ -545,9 +548,9 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) { unsigned i; - arena_stats_merge(arena, &cstats->dss, &cstats->pactive, - &cstats->pdirty, &cstats->astats, cstats->bstats, cstats->lstats, - cstats->hstats); + arena_stats_merge(arena, &cstats->dss, &cstats->lg_dirty_mult, + &cstats->pactive, &cstats->pdirty, &cstats->astats, cstats->bstats, + cstats->lstats, cstats->hstats); for (i = 0; i < NBINS; i++) { cstats->allocated_small += cstats->bstats[i].curregs * @@ -2000,6 +2003,8 @@ CTL_RO_CGEN(config_stats, stats_resident, ctl_stats.resident, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) +CTL_RO_GEN(stats_arenas_i_lg_dirty_mult, ctl_stats.arenas[mib[2]].lg_dirty_mult, + ssize_t) CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) diff --git a/src/stats.c b/src/stats.c index c5cea5e6..6e1752ef 100644 --- a/src/stats.c +++ b/src/stats.c @@ -6,15 +6,6 @@ xmallctl(n, v, &sz, NULL, 0); \ } while (0) -#define CTL_M1_GET(n, i, v, t) do { \ - size_t mib[6]; \ - size_t miblen = sizeof(mib) / sizeof(size_t); \ - size_t sz = sizeof(t); \ - xmallctlnametomib(n, mib, &miblen); \ - mib[1] = (i); \ - xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ -} while (0) - #define CTL_M2_GET(n, i, v, t) do { \ size_t mib[6]; \ size_t miblen = sizeof(mib) / sizeof(size_t); \ @@ -285,7 +276,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_M2_GET("stats.arenas.0.dss", i, &dss, const char *); malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n", dss); - CTL_M1_GET("arena.0.lg_dirty_mult", i, &lg_dirty_mult, ssize_t); + CTL_M2_GET("stats.arenas.0.lg_dirty_mult", i, &lg_dirty_mult, ssize_t); if (lg_dirty_mult >= 0) { malloc_cprintf(write_cb, cbopaque, "min active:dirty page ratio: %u:1\n", From 65db63cf3f0c5dd5126a1b3786756486eaf931ba Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Wed, 25 Mar 2015 18:56:55 -0700 Subject: [PATCH 271/352] Fix in-place shrinking huge reallocation purging bugs. Fix the shrinking case of huge_ralloc_no_move_similar() to purge the correct number of pages, at the correct offset. This regression was introduced by 8d6a3e8321a7767cb2ca0930b85d5d488a8cc659 (Implement dynamic per arena control over dirty page purging.). Fix huge_ralloc_no_move_shrink() to purge the correct number of pages. This bug was introduced by 9673983443a0782d975fbcb5d8457cfd411b8b56 (Purge/zero sub-chunk huge allocations as necessary.). --- src/arena.c | 7 +------ src/huge.c | 31 ++++++++++++++++--------------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/arena.c b/src/arena.c index bc13d209..30410683 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1245,16 +1245,11 @@ arena_purge_stashed(arena_t *arena, if (rdelm == &chunkselm->rd) { size_t size = extent_node_size_get(chunkselm); - void *addr, *chunk; - size_t offset; bool unzeroed; npages = size >> LG_PAGE; - addr = extent_node_addr_get(chunkselm); - chunk = CHUNK_ADDR2BASE(addr); - offset = CHUNK_ADDR2OFFSET(addr); unzeroed = chunk_purge_wrapper(arena, chunk_purge, - chunk, offset, size); + extent_node_addr_get(chunkselm), 0, size); extent_node_zeroed_set(chunkselm, !unzeroed); chunkselm = qr_next(chunkselm, cc_link); } else { diff --git a/src/huge.c b/src/huge.c index aa26f5df..32af2058 100644 --- a/src/huge.c +++ b/src/huge.c @@ -145,12 +145,11 @@ huge_ralloc_no_move_similar(void *ptr, size_t oldsize, size_t usize, /* Fill if necessary (shrinking). */ if (oldsize > usize) { - size_t sdiff = CHUNK_CEILING(usize) - usize; - zeroed = (sdiff != 0) ? !chunk_purge_wrapper(arena, chunk_purge, - CHUNK_ADDR2BASE(ptr), CHUNK_ADDR2OFFSET(ptr), usize) : true; + size_t sdiff = oldsize - usize; + zeroed = !chunk_purge_wrapper(arena, chunk_purge, ptr, usize, + sdiff); if (config_fill && unlikely(opt_junk_free)) { - memset((void *)((uintptr_t)ptr + usize), 0x5a, oldsize - - usize); + memset((void *)((uintptr_t)ptr + usize), 0x5a, sdiff); zeroed = false; } } else @@ -186,7 +185,6 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) extent_node_t *node; arena_t *arena; chunk_purge_t *chunk_purge; - size_t sdiff; bool zeroed; node = huge_node_get(ptr); @@ -196,15 +194,18 @@ huge_ralloc_no_move_shrink(void *ptr, size_t oldsize, size_t usize) chunk_purge = arena->chunk_purge; malloc_mutex_unlock(&arena->lock); - sdiff = CHUNK_CEILING(usize) - usize; - zeroed = (sdiff != 0) ? !chunk_purge_wrapper(arena, chunk_purge, - CHUNK_ADDR2BASE((uintptr_t)ptr + usize), - CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff) : true; - if (config_fill && unlikely(opt_junk_free)) { - huge_dalloc_junk((void *)((uintptr_t)ptr + usize), oldsize - - usize); - zeroed = false; - } + if (oldsize > usize) { + size_t sdiff = oldsize - usize; + zeroed = !chunk_purge_wrapper(arena, chunk_purge, + CHUNK_ADDR2BASE((uintptr_t)ptr + usize), + CHUNK_ADDR2OFFSET((uintptr_t)ptr + usize), sdiff); + if (config_fill && unlikely(opt_junk_free)) { + huge_dalloc_junk((void *)((uintptr_t)ptr + usize), + sdiff); + zeroed = false; + } + } else + zeroed = true; malloc_mutex_lock(&arena->huge_mtx); /* Update the size of the huge allocation. */ From b80fbcbbdb7ea6ba5918db7c665c836baa8c0b2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Marie?= Date: Tue, 7 Apr 2015 12:21:19 +0200 Subject: [PATCH 272/352] OpenBSD don't support TLS under some compiler (gcc 4.8.4 in particular), the auto-detection of TLS don't work properly. force tls to be disabled. the testsuite pass under gcc (4.8.4) and gcc (4.2.1) --- configure.ac | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index be49743d..bf2ac3a8 100644 --- a/configure.ac +++ b/configure.ac @@ -283,7 +283,13 @@ case "${host}" in abi="elf" AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) ;; - *-*-openbsd*|*-*-bitrig*) + *-*-openbsd*) + CFLAGS="$CFLAGS" + abi="elf" + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + force_tls="0" + ;; + *-*-bitrig*) CFLAGS="$CFLAGS" abi="elf" AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) From 897503521ddb703a1388899f79112e048c328278 Mon Sep 17 00:00:00 2001 From: Qinfan Wu Date: Tue, 21 Apr 2015 16:57:42 -0700 Subject: [PATCH 273/352] Fix mallctl doc: arenas.hchunk..size --- ChangeLog | 2 +- doc/jemalloc.xml.in | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ChangeLog b/ChangeLog index 8cc214ab..6f79cacf 100644 --- a/ChangeLog +++ b/ChangeLog @@ -47,7 +47,7 @@ found in the git revision history: + The "stats.arenas..huge.allocated", "stats.arenas..huge.nmalloc", "stats.arenas..huge.ndalloc", and "stats.arenas..huge.nrequests" mallctls provide high level per arena huge allocation statistics. - + The "arenas.nhchunks", "arenas.hchunks..size", + + The "arenas.nhchunks", "arenas.hchunk..size", "stats.arenas..hchunks..nmalloc", "stats.arenas..hchunks..ndalloc", "stats.arenas..hchunks..nrequests", and diff --git a/doc/jemalloc.xml.in b/doc/jemalloc.xml.in index d3f36164..c9ee9970 100644 --- a/doc/jemalloc.xml.in +++ b/doc/jemalloc.xml.in @@ -1756,9 +1756,9 @@ malloc_conf = "xmalloc:true";]]> Total number of huge size classes. - + - arenas.hchunks.<i>.size + arenas.hchunk.<i>.size (size_t) r- From 95e88de0aab257020dfc33248b86331cbfac28b1 Mon Sep 17 00:00:00 2001 From: Igor Podlesny Date: Tue, 24 Mar 2015 12:49:26 +0700 Subject: [PATCH 274/352] Concise JEMALLOC_HAVE_ISSETUGID case in secure_getenv(). --- src/jemalloc.c | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/jemalloc.c b/src/jemalloc.c index 7e9f4860..a2d1c5c2 100644 --- a/src/jemalloc.c +++ b/src/jemalloc.c @@ -709,24 +709,16 @@ stats_print_atexit(void) */ #ifndef JEMALLOC_HAVE_SECURE_GETENV +static char * +secure_getenv(const char *name) +{ + # ifdef JEMALLOC_HAVE_ISSETUGID -static char * -secure_getenv(const char *name) -{ - - if (issetugid() == 0) - return (getenv(name)); - else + if (issetugid() != 0) return (NULL); -} -# else -static char * -secure_getenv(const char *name) -{ - +# endif return (getenv(name)); } -# endif #endif static unsigned From f1f2b4542902c5bc14788f6c2d4190b422e5901f Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 1 May 2015 08:57:41 -0700 Subject: [PATCH 275/352] Embed full library install when running ld on OS X. This resolves #228. --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index bf2ac3a8..5f9bbd37 100644 --- a/configure.ac +++ b/configure.ac @@ -268,7 +268,7 @@ case "${host}" in so="dylib" importlib="${so}" force_tls="0" - DSO_LDFLAGS='-shared -Wl,-dylib_install_name,$(@F)' + DSO_LDFLAGS='-shared -Wl,-install_name,$(LIBDIR)/$(@F)' SOREV="${rev}.${so}" sbrk_deprecated="1" ;; From 8e33c21d2d03ee7f540e32c3d75b10c128eaea57 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 1 May 2015 09:03:20 -0700 Subject: [PATCH 276/352] Prefer /proc//task//maps over /proc//maps on Linux. This resolves #227. --- src/prof.c | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/prof.c b/src/prof.c index f2a37253..8453ea87 100644 --- a/src/prof.c +++ b/src/prof.c @@ -1338,21 +1338,40 @@ label_return: return (ret); } +JEMALLOC_ATTR(format(printf, 1, 2)) +static int +prof_open_maps(const char *format, ...) +{ + int mfd; + va_list ap; + char filename[PATH_MAX + 1]; + + va_start(ap, format); + malloc_vsnprintf(filename, sizeof(filename), format, ap); + va_end(ap); + mfd = open(filename, O_RDONLY); + + return (mfd); +} + static bool prof_dump_maps(bool propagate_err) { bool ret; int mfd; - char filename[PATH_MAX + 1]; cassert(config_prof); #ifdef __FreeBSD__ - malloc_snprintf(filename, sizeof(filename), "/proc/curproc/map"); + mfd = prof_open_maps("/proc/curproc/map"); #else - malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps", - (int)getpid()); + { + int pid = getpid(); + + mfd = prof_open_maps("/proc/%d/task/%d/maps", pid, pid); + if (mfd == -1) + mfd = prof_open_maps("/proc/%d/maps", pid); + } #endif - mfd = open(filename, O_RDONLY); if (mfd != -1) { ssize_t nread; From 7041720ac208fa2f7f65e40d8133d4b291516847 Mon Sep 17 00:00:00 2001 From: Jason Evans Date: Fri, 1 May 2015 12:31:12 -0700 Subject: [PATCH 277/352] Rename pprof to jeprof. This rename avoids installation collisions with the upstream gperftools. Additionally, jemalloc's per thread heap profile functionality introduced an incompatible file format, so it's now worthwhile to clearly distinguish jemalloc's version of this script from the upstream version. This resolves #229. --- .gitignore | 1 + ChangeLog | 5 +- Makefile.in | 2 +- bin/{pprof => jeprof.in} | 114 ++++++++++++++++++++------------------- configure.ac | 2 +- doc/jemalloc.xml.in | 5 +- src/prof.c | 2 +- 7 files changed, 68 insertions(+), 63 deletions(-) rename bin/{pprof => jeprof.in} (98%) mode change 100755 => 100644 diff --git a/.gitignore b/.gitignore index 5cd3e922..d0e39361 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /bin/jemalloc-config /bin/jemalloc.sh +/bin/jeprof /config.stamp /config.log diff --git a/ChangeLog b/ChangeLog index 6f79cacf..33139f9b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -117,8 +117,9 @@ found in the git revision history: - Assure that the constness of malloc_usable_size()'s return type matches that of the system implementation. - Change the heap profile dump format to support per thread heap profiling, - and enhance pprof with the --thread= option. As a result, the bundled - pprof must now be used rather than the upstream (gperftools) pprof. + rename pprof to jeprof, and enhance it with the --thread= option. As a + result, the bundled jeprof must now be used rather than the upstream + (gperftools) pprof. - Disable "opt.prof_final" by default, in order to avoid atexit(3), which can internally deadlock on some platforms. - Change the "arenas.nlruns" mallctl type from size_t to unsigned. diff --git a/Makefile.in b/Makefile.in index a105bb12..f539fad6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -73,7 +73,7 @@ endif LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix) # Lists of files. -BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh +BINS := $(objroot)bin/jemalloc-config $(objroot)bin/jemalloc.sh $(objroot)bin/jeprof C_HDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h C_SRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c \ $(srcroot)src/atomic.c $(srcroot)src/base.c $(srcroot)src/bitmap.c \ diff --git a/bin/pprof b/bin/jeprof.in old mode 100755 new mode 100644 similarity index 98% rename from bin/pprof rename to bin/jeprof.in index df503aea..e7178078 --- a/bin/pprof +++ b/bin/jeprof.in @@ -40,28 +40,28 @@ # # Examples: # -# % tools/pprof "program" "profile" +# % tools/jeprof "program" "profile" # Enters "interactive" mode # -# % tools/pprof --text "program" "profile" +# % tools/jeprof --text "program" "profile" # Generates one line per procedure # -# % tools/pprof --gv "program" "profile" +# % tools/jeprof --gv "program" "profile" # Generates annotated call-graph and displays via "gv" # -# % tools/pprof --gv --focus=Mutex "program" "profile" +# % tools/jeprof --gv --focus=Mutex "program" "profile" # Restrict to code paths that involve an entry that matches "Mutex" # -# % tools/pprof --gv --focus=Mutex --ignore=string "program" "profile" +# % tools/jeprof --gv --focus=Mutex --ignore=string "program" "profile" # Restrict to code paths that involve an entry that matches "Mutex" # and does not match "string" # -# % tools/pprof --list=IBF_CheckDocid "program" "profile" +# % tools/jeprof --list=IBF_CheckDocid "program" "profile" # Generates disassembly listing of all routines with at least one # sample that match the --list= pattern. The listing is # annotated with the flat and cumulative sample counts at each line. # -# % tools/pprof --disasm=IBF_CheckDocid "program" "profile" +# % tools/jeprof --disasm=IBF_CheckDocid "program" "profile" # Generates disassembly listing of all routines with at least one # sample that match the --disasm= pattern. The listing is # annotated with the flat and cumulative sample counts at each PC value. @@ -72,10 +72,11 @@ use strict; use warnings; use Getopt::Long; +my $JEPROF_VERSION = "@jemalloc_version@"; my $PPROF_VERSION = "2.0"; # These are the object tools we use which can come from a -# user-specified location using --tools, from the PPROF_TOOLS +# user-specified location using --tools, from the JEPROF_TOOLS # environment variable, or from the environment. my %obj_tool_map = ( "objdump" => "objdump", @@ -144,13 +145,13 @@ my $sep_address = undef; sub usage_string { return < +jeprof [options] is a space separated list of profile names. -pprof [options] +jeprof [options] is a list of profile files where each file contains the necessary symbol mappings as well as profile data (likely generated with --raw). -pprof [options] +jeprof [options] is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE Each name can be: @@ -161,9 +162,9 @@ pprof [options] $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, $CENSUSPROFILE_PAGE, or /pprof/filteredprofile. For instance: - pprof http://myserver.com:80$HEAP_PAGE + jeprof http://myserver.com:80$HEAP_PAGE If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). -pprof --symbols +jeprof --symbols Maps addresses to symbol names. In this mode, stdin should be a list of library mappings, in the same format as is found in the heap- and cpu-profile files (this loosely matches that of /proc/self/maps @@ -202,7 +203,7 @@ Output type: --pdf Generate PDF to stdout --svg Generate SVG to stdout --gif Generate GIF to stdout - --raw Generate symbolized pprof data (useful with remote fetch) + --raw Generate symbolized jeprof data (useful with remote fetch) Heap-Profile Options: --inuse_space Display in-use (mega)bytes [default] @@ -236,34 +237,34 @@ Miscellaneous: --version Version information Environment Variables: - PPROF_TMPDIR Profiles directory. Defaults to \$HOME/pprof - PPROF_TOOLS Prefix for object tools pathnames + JEPROF_TMPDIR Profiles directory. Defaults to \$HOME/jeprof + JEPROF_TOOLS Prefix for object tools pathnames Examples: -pprof /bin/ls ls.prof +jeprof /bin/ls ls.prof Enters "interactive" mode -pprof --text /bin/ls ls.prof +jeprof --text /bin/ls ls.prof Outputs one line per procedure -pprof --web /bin/ls ls.prof +jeprof --web /bin/ls ls.prof Displays annotated call-graph in web browser -pprof --gv /bin/ls ls.prof +jeprof --gv /bin/ls ls.prof Displays annotated call-graph via 'gv' -pprof --gv --focus=Mutex /bin/ls ls.prof +jeprof --gv --focus=Mutex /bin/ls ls.prof Restricts to code paths including a .*Mutex.* entry -pprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof +jeprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof Code paths including Mutex but not string -pprof --list=getdir /bin/ls ls.prof +jeprof --list=getdir /bin/ls ls.prof (Per-line) annotated source listing for getdir() -pprof --disasm=getdir /bin/ls ls.prof +jeprof --disasm=getdir /bin/ls ls.prof (Per-PC) annotated disassembly for getdir() -pprof http://localhost:1234/ +jeprof http://localhost:1234/ Enters "interactive" mode -pprof --text localhost:1234 +jeprof --text localhost:1234 Outputs one line per procedure for localhost:1234 -pprof --raw localhost:1234 > ./local.raw -pprof --text ./local.raw +jeprof --raw localhost:1234 > ./local.raw +jeprof --text ./local.raw Fetches a remote profile for later analysis and then analyzes it in text mode. EOF @@ -271,7 +272,8 @@ EOF sub version_string { return <readline('(pprof) '))) { + my $term = new Term::ReadLine 'jeprof'; + while ( defined ($_ = $term->readline('(jeprof) '))) { $term->addhistory($_) if /\S/; if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { last; # exit when we get an interactive command to quit @@ -817,7 +819,7 @@ sub InteractiveMode { } } else { # don't have readline while (1) { - print STDERR "(pprof) "; + print STDERR "(jeprof) "; $_ = ; last if ! defined $_ ; s/\r//g; # turn windows-looking lines into unix-looking lines @@ -1010,7 +1012,7 @@ sub ProcessProfile { sub InteractiveHelpMessage { print STDERR <