jemalloc/test/unit/cache_bin.c

#include "test/jemalloc_test.h"

#ifdef JEMALLOC_EXPERIMENTAL_FASTPATH_PREFETCH
static bool experimental_fast_prefetch_enabled = true;
#else
static bool experimental_fast_prefetch_enabled = false;
#endif

static void
do_fill_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t ncached_max,
    cache_bin_sz_t nfill_attempt, cache_bin_sz_t nfill_succeed) {
	bool success;
	void *ptr;
	assert_true(cache_bin_ncached_get_local(bin) == 0, "");
	CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill_attempt);
	cache_bin_init_ptr_array_for_fill(bin, &arr, nfill_attempt);
	for (cache_bin_sz_t i = 0; i < nfill_succeed; i++) {
		arr.ptr[i] = &ptrs[i];
	}
	cache_bin_finish_fill(bin, &arr, nfill_succeed);
	expect_true(cache_bin_ncached_get_local(bin) == nfill_succeed,
	    "");
	cache_bin_low_water_set(bin);

	for (cache_bin_sz_t i = 0; i < nfill_succeed; i++) {
		ptr = cache_bin_alloc(bin, &success);
		expect_true(success, "");
		expect_ptr_eq(ptr, (void *)&ptrs[i],
		    "Should pop in order filled");
		expect_true(cache_bin_low_water_get(bin)
		    == nfill_succeed - i - 1, "");
	}
	expect_true(cache_bin_ncached_get_local(bin) == 0, "");
	expect_true(cache_bin_low_water_get(bin) == 0, "");
}

static void
do_flush_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
    cache_bin_sz_t nflush) {
	bool success;
	assert_true(cache_bin_ncached_get_local(bin) == 0, "");

	for (cache_bin_sz_t i = 0; i < nfill; i++) {
		success = cache_bin_dalloc_easy(bin, &ptrs[i]);
		expect_true(success, "");
	}

	CACHE_BIN_PTR_ARRAY_DECLARE(arr, nflush);
	cache_bin_init_ptr_array_for_flush(bin, &arr, nflush);
	for (cache_bin_sz_t i = 0; i < nflush; i++) {
		expect_ptr_eq(arr.ptr[i], &ptrs[nflush - i - 1], "");
	}
	cache_bin_finish_flush(bin, &arr, nflush);

	expect_true(cache_bin_ncached_get_local(bin) == nfill - nflush,
	    "");
	while (cache_bin_ncached_get_local(bin) > 0) {
		cache_bin_alloc(bin, &success);
	}
}

static void
do_batch_alloc_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
    size_t batch) {
	assert_true(cache_bin_ncached_get_local(bin) == 0, "");
	CACHE_BIN_PTR_ARRAY_DECLARE(arr, nfill);
	cache_bin_init_ptr_array_for_fill(bin, &arr, nfill);
	for (cache_bin_sz_t i = 0; i < nfill; i++) {
		arr.ptr[i] = &ptrs[i];
	}
	cache_bin_finish_fill(bin, &arr, nfill);
	assert_true(cache_bin_ncached_get_local(bin) == nfill, "");
	cache_bin_low_water_set(bin);

	void **out = malloc((batch + 1) * sizeof(void *));
	size_t n = cache_bin_alloc_batch(bin, batch, out);
	assert_true(n == ((size_t)nfill < batch ? (size_t)nfill : batch), "");
	for (cache_bin_sz_t i = 0; i < (cache_bin_sz_t)n; i++) {
		expect_ptr_eq(out[i], &ptrs[i], "");
	}
	expect_true(cache_bin_low_water_get(bin) == nfill -
	    (cache_bin_sz_t)n, "");
	while (cache_bin_ncached_get_local(bin) > 0) {
		bool success;
		cache_bin_alloc(bin, &success);
	}
	free(out);
}

static void
test_bin_init(cache_bin_t *bin, cache_bin_info_t *info) {
	size_t size;
	size_t alignment;
	cache_bin_info_compute_alloc(info, 1, &size, &alignment);
	void *mem = mallocx(size, MALLOCX_ALIGN(alignment));
	assert_ptr_not_null(mem, "Unexpected mallocx failure");

	size_t cur_offset = 0;
	cache_bin_preincrement(info, 1, mem, &cur_offset);
	cache_bin_init(bin, info, mem, &cur_offset);
	cache_bin_postincrement(mem, &cur_offset);
	assert_zu_eq(cur_offset, size, "Should use all requested memory");
}

TEST_BEGIN(test_cache_bin) {
	const int ncached_max = 100;
	bool success;
	void *ptr;

	cache_bin_info_t info;
	cache_bin_info_init(&info, ncached_max);
	cache_bin_t bin;
	test_bin_init(&bin, &info);

	/* Initialize to empty; should then have 0 elements. */
	expect_d_eq(ncached_max, cache_bin_ncached_max_get(&bin), "");
	expect_true(cache_bin_ncached_get_local(&bin) == 0, "");
	expect_true(cache_bin_low_water_get(&bin) == 0, "");

	ptr = cache_bin_alloc_easy(&bin, &success);
	expect_false(success, "Shouldn't successfully allocate when empty");
	expect_ptr_null(ptr, "Shouldn't get a non-null pointer on failure");

	ptr = cache_bin_alloc(&bin, &success);
	expect_false(success, "Shouldn't successfully allocate when empty");
	expect_ptr_null(ptr, "Shouldn't get a non-null pointer on failure");

	/*
	 * We allocate one more item than ncached_max, so we can test cache bin
	 * exhaustion.
	 */
	void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0);
	assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
	for  (cache_bin_sz_t i = 0; i < ncached_max; i++) {
		expect_true(cache_bin_ncached_get_local(&bin) == i, "");
		success = cache_bin_dalloc_easy(&bin, &ptrs[i]);
		expect_true(success,
		    "Should be able to dalloc into a non-full cache bin.");
		expect_true(cache_bin_low_water_get(&bin) == 0,
		    "Pushes and pops shouldn't change low water of zero.");
	}
	expect_true(cache_bin_ncached_get_local(&bin) == ncached_max,
	    "");
	success = cache_bin_dalloc_easy(&bin, &ptrs[ncached_max]);
	expect_false(success, "Shouldn't be able to dalloc into a full bin.");

	cache_bin_low_water_set(&bin);

	for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
		expect_true(cache_bin_low_water_get(&bin)
		    == ncached_max - i, "");
		expect_true(cache_bin_ncached_get_local(&bin)
		    == ncached_max - i, "");
		/*
		 * This should fail -- the easy variant can't change the low
		 * water mark.
		 */
		ptr = cache_bin_alloc_easy(&bin, &success);
		expect_ptr_null(ptr, "");
		expect_false(success, "");
		expect_true(cache_bin_low_water_get(&bin)
		    == ncached_max - i, "");
		expect_true(cache_bin_ncached_get_local(&bin)
		    == ncached_max - i, "");

		/* This should succeed, though. */
		ptr = cache_bin_alloc(&bin, &success);
		expect_true(success, "");
		expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1],
		    "Alloc should pop in stack order");
		expect_true(cache_bin_low_water_get(&bin)
		    == ncached_max - i - 1, "");
		expect_true(cache_bin_ncached_get_local(&bin)
		    == ncached_max - i - 1, "");
	}
	/* Now we're empty -- all alloc attempts should fail. */
	expect_true(cache_bin_ncached_get_local(&bin) == 0, "");
	ptr = cache_bin_alloc_easy(&bin, &success);
	expect_ptr_null(ptr, "");
	expect_false(success, "");
	ptr = cache_bin_alloc(&bin, &success);
	expect_ptr_null(ptr, "");
	expect_false(success, "");

	for (cache_bin_sz_t i = 0; i < ncached_max / 2; i++) {
		cache_bin_dalloc_easy(&bin, &ptrs[i]);
	}
	cache_bin_low_water_set(&bin);

	for (cache_bin_sz_t i = ncached_max / 2; i < ncached_max; i++) {
		cache_bin_dalloc_easy(&bin, &ptrs[i]);
	}
	expect_true(cache_bin_ncached_get_local(&bin) == ncached_max,
	    "");
	for (cache_bin_sz_t i = ncached_max - 1; i >= ncached_max / 2; i--) {
		/*
		 * Size is bigger than low water -- the reduced version should
		 * succeed.
		 */
		ptr = cache_bin_alloc_easy(&bin, &success);
		expect_true(success, "");
		expect_ptr_eq(ptr, &ptrs[i], "");
	}
	/* But now, we've hit low-water. */
	ptr = cache_bin_alloc_easy(&bin, &success);
	expect_false(success, "");
	expect_ptr_null(ptr, "");

	/* We're going to test filling -- we must be empty to start. */
	while (cache_bin_ncached_get_local(&bin)) {
		cache_bin_alloc(&bin, &success);
		expect_true(success, "");
	}

	/* Test fill. */
	/* Try to fill all, succeed fully. */
	do_fill_test(&bin, ptrs, ncached_max, ncached_max,
	    ncached_max);
	/* Try to fill all, succeed partially. */
	do_fill_test(&bin, ptrs, ncached_max, ncached_max,
	    ncached_max / 2);
	/* Try to fill all, fail completely. */
	do_fill_test(&bin, ptrs, ncached_max, ncached_max, 0);

	/* Try to fill some, succeed fully. */
	do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2,
	    ncached_max / 2);
	/* Try to fill some, succeed partially. */
	do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2,
	    ncached_max / 4);
	/* Try to fill some, fail completely. */
	do_fill_test(&bin, ptrs, ncached_max, ncached_max / 2, 0);

	do_flush_test(&bin, ptrs, ncached_max, ncached_max);
	do_flush_test(&bin, ptrs, ncached_max, ncached_max / 2);
	do_flush_test(&bin, ptrs, ncached_max, 0);
	do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 2);
	do_flush_test(&bin, ptrs, ncached_max / 2, ncached_max / 4);
	do_flush_test(&bin, ptrs, ncached_max / 2, 0);

	do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max);
	do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max * 2);
	do_batch_alloc_test(&bin, ptrs, ncached_max, ncached_max / 2);
	do_batch_alloc_test(&bin, ptrs, ncached_max, 2);
	do_batch_alloc_test(&bin, ptrs, ncached_max, 1);
	do_batch_alloc_test(&bin, ptrs, ncached_max, 0);
	do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 2);
	do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max);
	do_batch_alloc_test(&bin, ptrs, ncached_max / 2, ncached_max / 4);
	do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 2);
	do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 1);
	do_batch_alloc_test(&bin, ptrs, ncached_max / 2, 0);
	do_batch_alloc_test(&bin, ptrs, 2, ncached_max);
	do_batch_alloc_test(&bin, ptrs, 2, 2);
	do_batch_alloc_test(&bin, ptrs, 2, 1);
	do_batch_alloc_test(&bin, ptrs, 2, 0);
	do_batch_alloc_test(&bin, ptrs, 1, 2);
	do_batch_alloc_test(&bin, ptrs, 1, 1);
	do_batch_alloc_test(&bin, ptrs, 1, 0);
	do_batch_alloc_test(&bin, ptrs, 0, 2);
	do_batch_alloc_test(&bin, ptrs, 0, 1);
	do_batch_alloc_test(&bin, ptrs, 0, 0);

	free(ptrs);
}
TEST_END

static void
do_flush_stashed_test(cache_bin_t *bin, void **ptrs, cache_bin_sz_t nfill,
    cache_bin_sz_t nstash) {
	expect_true(cache_bin_ncached_get_local(bin) == 0,
	    "Bin not empty");
	expect_true(cache_bin_nstashed_get_local(bin) == 0,
	    "Bin not empty");
	expect_true(nfill + nstash <= bin->bin_info.ncached_max, "Exceeded max");

	bool ret;
	/* Fill */
	for (cache_bin_sz_t i = 0; i < nfill; i++) {
		ret = cache_bin_dalloc_easy(bin, &ptrs[i]);
		expect_true(ret, "Unexpected fill failure");
	}
	expect_true(cache_bin_ncached_get_local(bin) == nfill,
	    "Wrong cached count");

	/* Stash */
	for (cache_bin_sz_t i = 0; i < nstash; i++) {
		ret = cache_bin_stash(bin, &ptrs[i + nfill]);
		expect_true(ret, "Unexpected stash failure");
	}
	expect_true(cache_bin_nstashed_get_local(bin) == nstash,
	    "Wrong stashed count");

	if (nfill + nstash == bin->bin_info.ncached_max) {
		ret = cache_bin_dalloc_easy(bin, &ptrs[0]);
		expect_false(ret, "Should not dalloc into a full bin");
		ret = cache_bin_stash(bin, &ptrs[0]);
		expect_false(ret, "Should not stash into a full bin");
	}

	/* Alloc filled ones */
	for (cache_bin_sz_t i = 0; i < nfill; i++) {
		void *ptr = cache_bin_alloc(bin, &ret);
		expect_true(ret, "Unexpected alloc failure");
		/* Verify it's not from the stashed range. */
		expect_true((uintptr_t)ptr < (uintptr_t)&ptrs[nfill],
		    "Should not alloc stashed ptrs");
	}
	expect_true(cache_bin_ncached_get_local(bin) == 0,
	    "Wrong cached count");
	expect_true(cache_bin_nstashed_get_local(bin) == nstash,
	    "Wrong stashed count");

	cache_bin_alloc(bin, &ret);
	expect_false(ret, "Should not alloc stashed");

	/* Clear stashed ones */
	cache_bin_finish_flush_stashed(bin);
	expect_true(cache_bin_ncached_get_local(bin) == 0,
	    "Wrong cached count");
	expect_true(cache_bin_nstashed_get_local(bin) == 0,
	    "Wrong stashed count");

	cache_bin_alloc(bin, &ret);
	expect_false(ret, "Should not alloc from empty bin");
}

TEST_BEGIN(test_cache_bin_stash) {
	const int ncached_max = 100;

	cache_bin_t bin;
	cache_bin_info_t info;
	cache_bin_info_init(&info, ncached_max);
	test_bin_init(&bin, &info);

	/*
	 * The content of this array is not accessed; instead the interior
	 * addresses are used to insert / stash into the bins as test pointers.
	 */
	void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0);
	assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
	bool ret;
	for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
		expect_true(cache_bin_ncached_get_local(&bin) ==
		    (i / 2 + i % 2), "Wrong ncached value");
		expect_true(cache_bin_nstashed_get_local(&bin) ==
		    i / 2, "Wrong nstashed value");
		if (i % 2 == 0) {
			cache_bin_dalloc_easy(&bin, &ptrs[i]);
		} else {
			ret = cache_bin_stash(&bin, &ptrs[i]);
			expect_true(ret, "Should be able to stash into a "
			    "non-full cache bin");
		}
	}
	ret = cache_bin_dalloc_easy(&bin, &ptrs[0]);
	expect_false(ret, "Should not dalloc into a full cache bin");
	ret = cache_bin_stash(&bin, &ptrs[0]);
	expect_false(ret, "Should not stash into a full cache bin");
	for (cache_bin_sz_t i = 0; i < ncached_max; i++) {
		void *ptr = cache_bin_alloc(&bin, &ret);
		if (i < ncached_max / 2) {
			expect_true(ret, "Should be able to alloc");
			uintptr_t diff = ((uintptr_t)ptr - (uintptr_t)&ptrs[0])
			    / sizeof(void *);
			expect_true(diff % 2 == 0, "Should be able to alloc");
		} else {
			expect_false(ret, "Should not alloc stashed");
			expect_true(cache_bin_nstashed_get_local(&bin) == ncached_max / 2,
			    "Wrong nstashed value");
		}
	}

	test_bin_init(&bin, &info);
	do_flush_stashed_test(&bin, ptrs, ncached_max, 0);
	do_flush_stashed_test(&bin, ptrs, 0, ncached_max);
	do_flush_stashed_test(&bin, ptrs, ncached_max / 2,
	    ncached_max / 2);
	do_flush_stashed_test(&bin, ptrs, ncached_max / 4,
	    ncached_max / 2);
	do_flush_stashed_test(&bin, ptrs, ncached_max / 2,
	    ncached_max / 4);
	do_flush_stashed_test(&bin, ptrs, ncached_max / 4,
	    ncached_max / 4);
}
TEST_END

typedef struct {
	void *ptr;
	bool is_write;
} prefetch_arg_t;

#define PREFETCH_SZ 256
static prefetch_arg_t prefetch_calls[PREFETCH_SZ];
static unsigned nprefetch_calls;

static void
prefetch_hook(void *p, bool is_write) {
	prefetch_calls[nprefetch_calls].ptr = p;
	prefetch_calls[nprefetch_calls].is_write = is_write;
	++nprefetch_calls;
}

static void
reset_prefetch_calls(void) {
	nprefetch_calls = 0;
	cache_bin_prefetch_hook_set(prefetch_hook);
}

static void**
do_dallocs_allocs(cache_bin_t *bin, int ncached_max) {
	bool success = false;
	assert(ncached_max < PREFETCH_SZ);
	/*
	 * We allocate fully, so we can test
	 * prefetch at the end of the cache bin.
	 */
	void **ptrs = mallocx(sizeof(void *) * (ncached_max + 1), 0);
	assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
	for  (cache_bin_sz_t i = 0; i < ncached_max; i++) {
		expect_true(cache_bin_ncached_get_local(bin) == i, "");
		success = cache_bin_dalloc_easy(bin, &ptrs[i]);
	}
	expect_true(cache_bin_ncached_get_local(bin) == ncached_max,
	    "");

	reset_prefetch_calls();
	for  (cache_bin_sz_t i = 0; i < ncached_max; i++) {
		void *ptr = cache_bin_alloc_easy(bin, &success);
		expect_true(success, "");
		expect_ptr_eq(ptr, &ptrs[ncached_max - i - 1], "");
	}
	return ptrs;
}

TEST_BEGIN(test_cache_bin_alloc_easy_prefetch_disabled) {
	test_skip_if(experimental_fast_prefetch_enabled);

	const int ncached_max = 10;
	cache_bin_info_t info;
	cache_bin_info_init(&info, ncached_max);
	cache_bin_t bin;
	test_bin_init(&bin, &info);

	/* Initialize to empty; should then have 0 elements. */
	expect_d_eq(ncached_max, cache_bin_ncached_max_get(&bin), "");
	expect_true(cache_bin_ncached_get_local(&bin) == 0, "");

	void **ptrs = do_dallocs_allocs(&bin, ncached_max);
	/* Check prefetch calls */
	expect_zu_eq(nprefetch_calls, 0, "No calls when prefetch disabled");

	free(ptrs);
	cache_bin_prefetch_hook_set(NULL);
}
TEST_END

TEST_BEGIN(test_cache_bin_alloc_easy_prefetch_enabled) {
	test_skip_if(!experimental_fast_prefetch_enabled);
	const int ncached_max = 10;

	cache_bin_info_t info;
	cache_bin_info_init(&info, ncached_max);
	cache_bin_t bin;
	test_bin_init(&bin, &info);

	/* Initialize to empty; should then have 0 elements. */
	expect_d_eq(ncached_max, cache_bin_ncached_max_get(&bin), "");
	expect_true(cache_bin_ncached_get_local(&bin) == 0, "");

	void **ptrs = do_dallocs_allocs(&bin, ncached_max);
	/* Check prefetch calls */
	expect_zu_eq(nprefetch_calls, ncached_max, "Not enough prefetch calls");
	/*
	 * Each prefetched pointer should match one ahead in original array
	 * in the opposite order as bin's head moves backwards on allocations.
	 */
	for  (cache_bin_sz_t i = 1; i < ncached_max; i++) {
		expect_ptr_eq(prefetch_calls[i-1].ptr,
		    &ptrs[ncached_max - 1 - i], "prefetch address wrong");
	}

	/* Bin is empty now.  stack_head points one past the "real" slots */
	expect_true(cache_bin_ncached_get_local(&bin) == 0, "");
	void **expected_ptr = bin.stack_head;
	expect_ptr_eq(prefetch_calls[ncached_max - 1].ptr, expected_ptr,
		      "prefetch address wrong for out of boundary");
	expect_ptr_eq(expected_ptr, *expected_ptr, "Content is the address");

	free(ptrs);
	cache_bin_prefetch_hook_set(NULL);
}
TEST_END

int
main(void) {
	return test(test_cache_bin,
		test_cache_bin_stash,
		test_cache_bin_alloc_easy_prefetch_disabled,
		test_cache_bin_alloc_easy_prefetch_enabled);
}