Preserve forward scans in non-tree bitmap batches

Non-tree batch allocation restarts its bitmap search at group zero after
each group. Pass a lower bound to bitmap_ffu_group() so subsequent
iterations skip the exhausted prefix.

Keep the cursor local to each batch and assert that a group is exhausted
before continuing. Tree bitmaps retain their summary-based search.

include/jemalloc/internal/bitmap.h

@@ -366,15 +366,16 @@ bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) {
 
 /*
  * Index of the first bottom-level group that has an unset bit.  The bitmap must
- * not be full.
+ * not be full, and all groups before goff must be full.  This lower bound avoids
+ * rescanning full groups in a linear bitmap; a tree uses its summaries instead.
  */
 static inline size_t
-bitmap_ffu_group(const bitmap_t *bitmap, const bitmap_info_t *binfo) {
-	size_t goff = 0;
-
+bitmap_ffu_group(
+    const bitmap_t *bitmap, const bitmap_info_t *binfo, size_t goff) {
 	assert(!bitmap_full(bitmap, binfo));
 
 #ifdef BITMAP_USE_TREE
+	goff = 0;
 	for (unsigned i = binfo->nlevels - 1; i > 0; i--) {
 		bitmap_t g = bitmap[binfo->levels[i].group_offset + goff];
 		goff = (goff << LG_BITMAP_GROUP_NBITS) + ffs_lu(g);

src/bin.c

@@ -102,9 +102,10 @@ bin_slab_reg_alloc_batch(
 	uintptr_t base = (uintptr_t)edata_addr_get(slab);
 	uintptr_t regsize = (uintptr_t)bin_info->reg_size;
 	unsigned  i = 0;
+	size_t    group = 0;
 	while (i < cnt) {
-		size_t group = bitmap_ffu_group(
-		    slab_data->bitmap, &bin_info->bitmap_info);
+		group = bitmap_ffu_group(
+		    slab_data->bitmap, &bin_info->bitmap_info, group);
 		bitmap_t g = slab_data->bitmap[group];
 		size_t shift = group << LG_BITMAP_GROUP_NBITS;
 #if (!defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE)
@@ -129,6 +130,12 @@ bin_slab_reg_alloc_batch(
 		}
 		bitmap_group_set(
 		    slab_data->bitmap, &bin_info->bitmap_info, group, g);
+		/*
+		 * If the batch is incomplete, this group is exhausted.  The bin
+		 * lock prevents earlier groups from becoming available again.
+		 */
+		assert(i == cnt || g == 0);
+		group++;
 	}
 	edata_nfree_sub(slab, cnt);
 }

test/unit/bin.c

@@ -230,6 +230,80 @@ TEST_BEGIN(test_bin_slab_reg_alloc_batch_fragmented) {
 }
 TEST_END
 
+/*
+ * Exercise batch allocation across full groups, resuming a partial group, and
+ * finding a region freed before the previous batch's cursor.
+ */
+TEST_BEGIN(test_bin_slab_reg_alloc_batch_progress) {
+	for (szind_t binind = 0; binind < SC_NBINS; binind++) {
+		const bin_info_t *bin_info = &bin_infos[binind];
+		unsigned          nregs = bin_info->nregs;
+		edata_t           batch_slab, single_slab;
+		void            **ptrs = mallocx(nregs * sizeof(void *), 0);
+		assert_ptr_not_null(ptrs, "Unexpected mallocx failure");
+
+		create_mock_slab(&batch_slab, binind, 0);
+		bitmap_t *bitmap = edata_slab_data_get(&batch_slab)->bitmap;
+		bitmap_init(bitmap, &bin_info->bitmap_info, true);
+		edata_nfree_set(&batch_slab, 0);
+		/*
+		 * Leave leading and intervening groups full, with a few free
+		 * regions in the others.  Include the last region to exercise
+		 * the end of the bitmap, including a partial final group.
+		 */
+		for (unsigned i = 0; i < nregs; i++) {
+			unsigned group = i >> LG_BITMAP_GROUP_NBITS;
+			unsigned bit = i & BITMAP_GROUP_NBITS_MASK;
+			if ((group % 3 != 0
+			        && (bit < 2 || bit == BITMAP_GROUP_NBITS - 1))
+			    || i == nregs - 1) {
+				bitmap_unset(bitmap, &bin_info->bitmap_info, i);
+				edata_nfree_inc(&batch_slab);
+			}
+		}
+		/* Mock slabs share backing storage, but not their bitmaps. */
+		single_slab = batch_slab;
+		bitmap_t *single_bitmap =
+		    edata_slab_data_get(&single_slab)->bitmap;
+		unsigned counts[] = {0, 1, 2, nregs};
+		unsigned nbatches = sizeof(counts) / sizeof(counts[0]);
+		for (unsigned b = 0; b < nbatches; b++) {
+			if (b == 2) {
+				/* Restart at the first group on each call. */
+				bitmap_unset(bitmap, &bin_info->bitmap_info, 0);
+				edata_nfree_inc(&batch_slab);
+				bitmap_unset(single_bitmap,
+				    &bin_info->bitmap_info, 0);
+				edata_nfree_inc(&single_slab);
+			}
+			unsigned nfree = edata_nfree_get(&batch_slab);
+			if (nfree == 0) {
+				continue;
+			}
+			unsigned cnt = counts[b] < nfree ? counts[b] : nfree;
+			bin_slab_reg_alloc_batch(
+			    &batch_slab, bin_info, cnt, ptrs);
+			for (unsigned i = 0; i < cnt; i++) {
+				expect_ptr_eq(ptrs[i],
+				    bin_slab_reg_alloc(&single_slab, bin_info),
+				    "bin %u, batch %u: region %u differs",
+				    binind, b, i);
+			}
+			expect_u_eq(edata_nfree_get(&batch_slab),
+			    edata_nfree_get(&single_slab),
+			    "bin %u, batch %u: nfree differs", binind, b);
+			expect_d_eq(memcmp(bitmap, single_bitmap,
+			                bitmap_size(&bin_info->bitmap_info)),
+			    0, "bin %u, batch %u: bitmaps differ", binind, b);
+		}
+		expect_u_eq(edata_nfree_get(&batch_slab), 0,
+		    "Free regions remain for bin %u", binind);
+		free(ptrs);
+		free(edata_addr_get(&batch_slab));
+	}
+}
+TEST_END
+
 /*
  * Test nonfull slab list insert, remove, and tryget.
  */
@@ -959,6 +1033,7 @@ main(void) {
 	    test_bin_slab_reg_alloc_batch,
 	    test_bin_slab_reg_alloc_batch_partial,
 	    test_bin_slab_reg_alloc_batch_fragmented,
+	    test_bin_slab_reg_alloc_batch_progress,
 	    test_bin_slabs_nonfull,
 	    test_bin_slabs_full,
 	    test_bin_slabs_full_auto,

test/unit/bitmap.c

@@ -343,8 +343,42 @@ TEST_BEGIN(test_bitmap_xfu) {
 }
 TEST_END
 
+TEST_BEGIN(test_bitmap_ffu_group) {
+	const size_t sizes[] = {1, BITMAP_GROUP_NBITS - 1, BITMAP_GROUP_NBITS,
+	    BITMAP_GROUP_NBITS + 1, BITMAP_MAXBITS - 1, BITMAP_MAXBITS};
+	for (unsigned i = 0; i < sizeof(sizes) / sizeof(sizes[0]); i++) {
+		bitmap_info_t binfo;
+		bitmap_info_init(&binfo, sizes[i]);
+		bitmap_t *bitmap = malloc(bitmap_size(&binfo));
+		assert_ptr_not_null(bitmap, "Unexpected malloc() failure");
+		bitmap_init(bitmap, &binfo, true);
+		for (size_t group = 0; group < BITMAP_BITS2GROUPS(sizes[i]);
+		     group++) {
+			/* Leave one free bit, including in a partial last group. */
+			size_t bit = (group + 1) * BITMAP_GROUP_NBITS - 1;
+			if (bit >= sizes[i]) {
+				bit = sizes[i] - 1;
+			}
+			bitmap_unset(bitmap, &binfo, bit);
+			/* Every lower bound has a full prefix, as required. */
+			size_t starts[] = {0, group / 2, group};
+			for (unsigned j = 0;
+			     j < sizeof(starts) / sizeof(starts[0]); j++) {
+				expect_zu_eq(
+				    bitmap_ffu_group(bitmap, &binfo, starts[j]),
+				    group, "nbits=%zu, start=%zu", sizes[i],
+				    starts[j]);
+			}
+			bitmap_set(bitmap, &binfo, bit);
+		}
+		free(bitmap);
+	}
+}
+TEST_END
+
 int
 main(void) {
 	return test(test_bitmap_initializer, test_bitmap_size, test_bitmap_init,
-	    test_bitmap_set, test_bitmap_unset, test_bitmap_xfu);
+	    test_bitmap_set, test_bitmap_unset, test_bitmap_xfu,
+	    test_bitmap_ffu_group);
 }