From 0d9d5d553e4ff722a64570a2f7c97399e5d37963 Mon Sep 17 00:00:00 2001 From: YANG XIN Date: Thu, 8 Oct 2026 16:20:02 +0800 Subject: [PATCH] Preserve forward scans in non-tree bitmap batches Non-tree batch allocation restarts its bitmap search at group zero after each group. Pass a lower bound to bitmap_ffu_group() so subsequent iterations skip the exhausted prefix. Keep the cursor local to each batch and assert that a group is exhausted before continuing. Tree bitmaps retain their summary-based search. --- include/jemalloc/internal/bitmap.h | 9 ++-- src/bin.c | 11 ++++- test/unit/bin.c | 75 ++++++++++++++++++++++++++++++ test/unit/bitmap.c | 36 +++++++++++++- 4 files changed, 124 insertions(+), 7 deletions(-) diff --git a/include/jemalloc/internal/bitmap.h b/include/jemalloc/internal/bitmap.h index a246bcd6..c3cdd480 100644 --- a/include/jemalloc/internal/bitmap.h +++ b/include/jemalloc/internal/bitmap.h @@ -366,15 +366,16 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) { /* * Index of the first bottom-level group that has an unset bit. The bitmap must - * not be full. + * not be full, and all groups before goff must be full. This lower bound avoids + * rescanning full groups in a linear bitmap; a tree uses its summaries instead. */ static inline size_t -bitmap_ffu_group(const bitmap_t *bitmap, const bitmap_info_t *binfo) { - size_t goff = 0; - +bitmap_ffu_group( + const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t goff) { assert(!bitmap_full(bitmap, binfo)); #ifdef BITMAP_USE_TREE + goff = 0; for (unsigned i = binfo->nlevels - 1; i > 0; i--) { bitmap_t g = bitmap[binfo->levels[i].group_offset + goff]; goff = (goff << LG_BITMAP_GROUP_NBITS) + ffs_lu(g); diff --git a/src/bin.c b/src/bin.c index f0db812f..937cc8e8 100644 --- a/src/bin.c +++ b/src/bin.c @@ -102,9 +102,10 @@ bin_slab_reg_alloc_batch( uintptr_t base = (uintptr_t)edata_addr_get(slab); uintptr_t regsize = (uintptr_t)bin_info->reg_size; unsigned i = 0; + size_t group = 0; while (i < cnt) { - size_t group = bitmap_ffu_group( - slab_data->bitmap, &bin_info->bitmap_info); + group = bitmap_ffu_group( + slab_data->bitmap, &bin_info->bitmap_info, group); bitmap_t g = slab_data->bitmap[group]; size_t shift = group << LG_BITMAP_GROUP_NBITS; #if (!defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) @@ -129,6 +130,12 @@ bin_slab_reg_alloc_batch( } bitmap_group_set( slab_data->bitmap, &bin_info->bitmap_info, group, g); + /* + * If the batch is incomplete, this group is exhausted. The bin + * lock prevents earlier groups from becoming available again. + */ + assert(i == cnt || g == 0); + group++; } edata_nfree_sub(slab, cnt); } diff --git a/test/unit/bin.c b/test/unit/bin.c index caaffdaf..4a9435f8 100644 --- a/test/unit/bin.c +++ b/test/unit/bin.c @@ -230,6 +230,80 @@ TEST_BEGIN(test_bin_slab_reg_alloc_batch_fragmented) { } TEST_END +/* + * Exercise batch allocation across full groups, resuming a partial group, and + * finding a region freed before the previous batch's cursor. + */ +TEST_BEGIN(test_bin_slab_reg_alloc_batch_progress) { + for (szind_t binind = 0; binind < SC_NBINS; binind++) { + const bin_info_t *bin_info = &bin_infos[binind]; + unsigned nregs = bin_info->nregs; + edata_t batch_slab, single_slab; + void **ptrs = mallocx(nregs * sizeof(void *), 0); + assert_ptr_not_null(ptrs, "Unexpected mallocx failure"); + + create_mock_slab(&batch_slab, binind, 0); + bitmap_t *bitmap = edata_slab_data_get(&batch_slab)->bitmap; + bitmap_init(bitmap, &bin_info->bitmap_info, true); + edata_nfree_set(&batch_slab, 0); + /* + * Leave leading and intervening groups full, with a few free + * regions in the others. Include the last region to exercise + * the end of the bitmap, including a partial final group. + */ + for (unsigned i = 0; i < nregs; i++) { + unsigned group = i >> LG_BITMAP_GROUP_NBITS; + unsigned bit = i & BITMAP_GROUP_NBITS_MASK; + if ((group % 3 != 0 + && (bit < 2 || bit == BITMAP_GROUP_NBITS - 1)) + || i == nregs - 1) { + bitmap_unset(bitmap, &bin_info->bitmap_info, i); + edata_nfree_inc(&batch_slab); + } + } + /* Mock slabs share backing storage, but not their bitmaps. */ + single_slab = batch_slab; + bitmap_t *single_bitmap = + edata_slab_data_get(&single_slab)->bitmap; + unsigned counts[] = {0, 1, 2, nregs}; + unsigned nbatches = sizeof(counts) / sizeof(counts[0]); + for (unsigned b = 0; b < nbatches; b++) { + if (b == 2) { + /* Restart at the first group on each call. */ + bitmap_unset(bitmap, &bin_info->bitmap_info, 0); + edata_nfree_inc(&batch_slab); + bitmap_unset(single_bitmap, + &bin_info->bitmap_info, 0); + edata_nfree_inc(&single_slab); + } + unsigned nfree = edata_nfree_get(&batch_slab); + if (nfree == 0) { + continue; + } + unsigned cnt = counts[b] < nfree ? counts[b] : nfree; + bin_slab_reg_alloc_batch( + &batch_slab, bin_info, cnt, ptrs); + for (unsigned i = 0; i < cnt; i++) { + expect_ptr_eq(ptrs[i], + bin_slab_reg_alloc(&single_slab, bin_info), + "bin %u, batch %u: region %u differs", + binind, b, i); + } + expect_u_eq(edata_nfree_get(&batch_slab), + edata_nfree_get(&single_slab), + "bin %u, batch %u: nfree differs", binind, b); + expect_d_eq(memcmp(bitmap, single_bitmap, + bitmap_size(&bin_info->bitmap_info)), + 0, "bin %u, batch %u: bitmaps differ", binind, b); + } + expect_u_eq(edata_nfree_get(&batch_slab), 0, + "Free regions remain for bin %u", binind); + free(ptrs); + free(edata_addr_get(&batch_slab)); + } +} +TEST_END + /* * Test nonfull slab list insert, remove, and tryget. */ @@ -959,6 +1033,7 @@ main(void) { test_bin_slab_reg_alloc_batch, test_bin_slab_reg_alloc_batch_partial, test_bin_slab_reg_alloc_batch_fragmented, + test_bin_slab_reg_alloc_batch_progress, test_bin_slabs_nonfull, test_bin_slabs_full, test_bin_slabs_full_auto, diff --git a/test/unit/bitmap.c b/test/unit/bitmap.c index 7e26ee5e..2c62c3a3 100644 --- a/test/unit/bitmap.c +++ b/test/unit/bitmap.c @@ -343,8 +343,42 @@ TEST_BEGIN(test_bitmap_xfu) { } TEST_END +TEST_BEGIN(test_bitmap_ffu_group) { + const size_t sizes[] = {1, BITMAP_GROUP_NBITS - 1, BITMAP_GROUP_NBITS, + BITMAP_GROUP_NBITS + 1, BITMAP_MAXBITS - 1, BITMAP_MAXBITS}; + for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, sizes[i]); + bitmap_t *bitmap = malloc(bitmap_size(&binfo)); + assert_ptr_not_null(bitmap, "Unexpected malloc() failure"); + bitmap_init(bitmap, &binfo, true); + for (size_t group = 0; group < BITMAP_BITS2GROUPS(sizes[i]); + group++) { + /* Leave one free bit, including in a partial last group. */ + size_t bit = (group + 1) * BITMAP_GROUP_NBITS - 1; + if (bit >= sizes[i]) { + bit = sizes[i] - 1; + } + bitmap_unset(bitmap, &binfo, bit); + /* Every lower bound has a full prefix, as required. */ + size_t starts[] = {0, group / 2, group}; + for (unsigned j = 0; + j < sizeof(starts) / sizeof(starts[0]); j++) { + expect_zu_eq( + bitmap_ffu_group(bitmap, &binfo, starts[j]), + group, "nbits=%zu, start=%zu", sizes[i], + starts[j]); + } + bitmap_set(bitmap, &binfo, bit); + } + free(bitmap); + } +} +TEST_END + int main(void) { return test(test_bitmap_initializer, test_bitmap_size, test_bitmap_init, - test_bitmap_set, test_bitmap_unset, test_bitmap_xfu); + test_bitmap_set, test_bitmap_unset, test_bitmap_xfu, + test_bitmap_ffu_group); }