Normalize benchmarks

- Favor external repetitions over manual repetitions and mintime when
possible.
- Add cmake ARROW_BUILD_BENCHMARKS_REFERENCE to toggle reference benchmarks.
- Remove default benchmark filter of `^Regression`.
- Remove Regression prefix from benchmark

cpp/CMakeLists.txt

@@ -269,6 +269,9 @@ else()
   add_custom_target(all-benchmarks)
   add_custom_target(benchmark ctest -L benchmark)
   add_dependencies(benchmark all-benchmarks)
+  if(ARROW_BUILD_BENCHMARKS_REFERENCE)
+    add_definitions(-DARROW_WITH_BENCHMARKS_REFERENCE)
+  endif()
 endif()
 
 if(NOT ARROW_BUILD_EXAMPLES)

cpp/cmake_modules/DefineOptions.cmake

@@ -96,6 +96,12 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   define_option(ARROW_BUILD_BENCHMARKS "Build the Arrow micro benchmarks, default OFF"
                 OFF)
 
+  # Reference benchmarks are used to compare to naive implementation, or
+  # discover various hardware limits.
+  define_option(ARROW_BUILD_BENCHMARKS_REFERENCE
+                "Build the Arrow micro reference benchmarks, default OFF."
+                OFF)
+
   define_option_string(ARROW_TEST_LINKAGE
                        "Linkage of Arrow libraries with unit tests executables."
                        "shared"

cpp/src/arrow/builder-benchmark.cc

@@ -70,7 +70,7 @@ static void ReferenceBuildVectorNoNulls(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildPrimitiveArrayNoNulls(
+static void BuildPrimitiveArrayNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   for (auto _ : state) {
     Int64Builder builder;
@@ -86,7 +86,7 @@ static void RegressionBuildPrimitiveArrayNoNulls(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildAdaptiveIntNoNulls(
+static void BuildAdaptiveIntNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   for (auto _ : state) {
     AdaptiveIntBuilder builder;
@@ -102,7 +102,7 @@ static void RegressionBuildAdaptiveIntNoNulls(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildAdaptiveIntNoNullsScalarAppend(
+static void BuildAdaptiveIntNoNullsScalarAppend(
     benchmark::State& state) {  // NOLINT non-const reference
   for (auto _ : state) {
     AdaptiveIntBuilder builder;
@@ -120,7 +120,7 @@ static void RegressionBuildAdaptiveIntNoNullsScalarAppend(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildBooleanArrayNoNulls(
+static void BuildBooleanArrayNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
 
   size_t n_bytes = kData.size() * sizeof(ValueType);
@@ -140,7 +140,7 @@ static void RegressionBuildBooleanArrayNoNulls(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildBinaryArray(
+static void BuildBinaryArray(
     benchmark::State& state) {  // NOLINT non-const reference
   for (auto _ : state) {
     BinaryBuilder builder;
@@ -156,7 +156,7 @@ static void RegressionBuildBinaryArray(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildChunkedBinaryArray(
+static void BuildChunkedBinaryArray(
     benchmark::State& state) {  // NOLINT non-const reference
   // 1MB chunks
   const int32_t kChunkSize = 1 << 20;
@@ -175,7 +175,7 @@ static void RegressionBuildChunkedBinaryArray(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildFixedSizeBinaryArray(
+static void BuildFixedSizeBinaryArray(
     benchmark::State& state) {  // NOLINT non-const reference
   auto type = fixed_size_binary(kBinaryView.size());
 
@@ -306,25 +306,25 @@ static void BenchmarkScalarDictionaryArray(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void RegressionBuildInt64DictionaryArrayRandom(
+static void BuildInt64DictionaryArrayRandom(
     benchmark::State& state) {  // NOLINT non-const reference
   const auto fodder = MakeRandomIntDictFodder();
   BenchmarkScalarDictionaryArray<DictionaryBuilder<Int64Type>>(state, fodder);
 }
 
-static void RegressionBuildInt64DictionaryArraySequential(
+static void BuildInt64DictionaryArraySequential(
     benchmark::State& state) {  // NOLINT non-const reference
   const auto fodder = MakeSequentialIntDictFodder();
   BenchmarkScalarDictionaryArray<DictionaryBuilder<Int64Type>>(state, fodder);
 }
 
-static void RegressionBuildInt64DictionaryArraySimilar(
+static void BuildInt64DictionaryArraySimilar(
     benchmark::State& state) {  // NOLINT non-const reference
   const auto fodder = MakeSimilarIntDictFodder();
   BenchmarkScalarDictionaryArray<DictionaryBuilder<Int64Type>>(state, fodder);
 }
 
-static void RegressionBuildStringDictionaryArray(
+static void BuildStringDictionaryArray(
     benchmark::State& state) {  // NOLINT non-const reference
   const auto fodder = MakeStringDictFodder();
   auto fodder_size =
@@ -347,7 +347,7 @@ static void RegressionBuildStringDictionaryArray(
   state.SetBytesProcessed(state.iterations() * fodder_size * kFinalSize);
 }
 
-static void BM_ArrayDataConstructDestruct(
+static void ArrayDataConstructDestruct(
     benchmark::State& state) {  // NOLINT non-const reference
   std::vector<std::shared_ptr<ArrayData>> arrays;
 
@@ -370,21 +370,21 @@ static void BM_ArrayDataConstructDestruct(
 
 BENCHMARK(ReferenceBuildVectorNoNulls);
 
-BENCHMARK(RegressionBuildBooleanArrayNoNulls);
+BENCHMARK(BuildBooleanArrayNoNulls);
 
-BENCHMARK(RegressionBuildPrimitiveArrayNoNulls);
-BENCHMARK(RegressionBuildAdaptiveIntNoNulls);
-BENCHMARK(RegressionBuildAdaptiveIntNoNullsScalarAppend);
+BENCHMARK(BuildPrimitiveArrayNoNulls);
+BENCHMARK(BuildAdaptiveIntNoNulls);
+BENCHMARK(BuildAdaptiveIntNoNullsScalarAppend);
 
-BENCHMARK(RegressionBuildBinaryArray);
-BENCHMARK(RegressionBuildChunkedBinaryArray);
-BENCHMARK(RegressionBuildFixedSizeBinaryArray);
+BENCHMARK(BuildBinaryArray);
+BENCHMARK(BuildChunkedBinaryArray);
+BENCHMARK(BuildFixedSizeBinaryArray);
 
-BENCHMARK(RegressionBuildInt64DictionaryArrayRandom);
-BENCHMARK(RegressionBuildInt64DictionaryArraySequential);
-BENCHMARK(RegressionBuildInt64DictionaryArraySimilar);
-BENCHMARK(RegressionBuildStringDictionaryArray);
+BENCHMARK(BuildInt64DictionaryArrayRandom);
+BENCHMARK(BuildInt64DictionaryArraySequential);
+BENCHMARK(BuildInt64DictionaryArraySimilar);
+BENCHMARK(BuildStringDictionaryArray);
 
-BENCHMARK(BM_ArrayDataConstructDestruct);
+BENCHMARK(ArrayDataConstructDestruct);
 
 }  // namespace arrow

cpp/src/arrow/compute/compute-benchmark.cc

@@ -198,19 +198,14 @@ static void BM_UniqueString100bytes(benchmark::State& state) {
   BenchUnique(state, HashParams<StringType>{0.05, 100}, state.range(0), state.range(1));
 }
 
-BENCHMARK(BM_BuildDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_BuildStringDictionary)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_BuildDictionary);
+BENCHMARK(BM_BuildStringDictionary);
 
-constexpr int kHashBenchmarkLength = 1 << 24;
+constexpr int kHashBenchmarkLength = 1 << 22;
 
 #define ADD_HASH_ARGS(WHAT)                        \
-  WHAT->Args({kHashBenchmarkLength, 50})           \
-      ->Args({kHashBenchmarkLength, 1 << 10})      \
+  WHAT->Args({kHashBenchmarkLength, 1 << 10})      \
       ->Args({kHashBenchmarkLength, 10 * 1 << 10}) \
-      ->Args({kHashBenchmarkLength, 1 << 20})      \
-      ->MinTime(1.0)                               \
-      ->Unit(benchmark::kMicrosecond)              \
-      ->UseRealTime()
 
 ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64NoNulls));
 ADD_HASH_ARGS(BENCHMARK(BM_UniqueInt64WithNulls));
@@ -219,15 +214,11 @@ ADD_HASH_ARGS(BENCHMARK(BM_UniqueString100bytes));
 
 BENCHMARK(BM_UniqueUInt8NoNulls)
     ->Args({kHashBenchmarkLength, 200})
-    ->MinTime(1.0)
-    ->Unit(benchmark::kMicrosecond)
-    ->UseRealTime();
+    ->Unit(benchmark::kMicrosecond);
 
 BENCHMARK(BM_UniqueUInt8WithNulls)
     ->Args({kHashBenchmarkLength, 200})
-    ->MinTime(1.0)
-    ->Unit(benchmark::kMicrosecond)
-    ->UseRealTime();
+    ->Unit(benchmark::kMicrosecond);
 
 }  // namespace compute
 }  // namespace arrow

cpp/src/arrow/compute/kernels/aggregate-benchmark.cc

@@ -37,6 +37,8 @@ namespace compute {
 #include <iostream>
 #include <random>
 
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
 namespace BitUtil = arrow::BitUtil;
 using arrow::internal::BitmapReader;
 
@@ -273,7 +275,7 @@ struct SumBitmapVectorizeUnroll : public Summer<T> {
 };
 
 template <typename Functor>
-void BenchSum(benchmark::State& state) {
+void ReferenceSum(benchmark::State& state) {
   using T = typename Functor::ValueType;
 
   const int64_t array_size = state.range(0) / sizeof(int64_t);
@@ -295,15 +297,17 @@ void BenchSum(benchmark::State& state) {
   state.SetBytesProcessed(state.iterations() * array_size * sizeof(T));
 }
 
-BENCHMARK_TEMPLATE(BenchSum, SumNoNulls<int64_t>)->Apply(BenchmarkSetArgs);
-BENCHMARK_TEMPLATE(BenchSum, SumNoNullsUnrolled<int64_t>)->Apply(BenchmarkSetArgs);
-BENCHMARK_TEMPLATE(BenchSum, SumSentinel<int64_t>)->Apply(BenchmarkSetArgs);
-BENCHMARK_TEMPLATE(BenchSum, SumSentinelUnrolled<int64_t>)->Apply(BenchmarkSetArgs);
-BENCHMARK_TEMPLATE(BenchSum, SumBitmapNaive<int64_t>)->Apply(BenchmarkSetArgs);
-BENCHMARK_TEMPLATE(BenchSum, SumBitmapReader<int64_t>)->Apply(BenchmarkSetArgs);
-BENCHMARK_TEMPLATE(BenchSum, SumBitmapVectorizeUnroll<int64_t>)->Apply(BenchmarkSetArgs);
+BENCHMARK_TEMPLATE(ReferenceSum, SumNoNulls<int64_t>)->Apply(BenchmarkSetArgs);
+BENCHMARK_TEMPLATE(ReferenceSum, SumNoNullsUnrolled<int64_t>)->Apply(BenchmarkSetArgs);
+BENCHMARK_TEMPLATE(ReferenceSum, SumSentinel<int64_t>)->Apply(BenchmarkSetArgs);
+BENCHMARK_TEMPLATE(ReferenceSum, SumSentinelUnrolled<int64_t>)->Apply(BenchmarkSetArgs);
+BENCHMARK_TEMPLATE(ReferenceSum, SumBitmapNaive<int64_t>)->Apply(BenchmarkSetArgs);
+BENCHMARK_TEMPLATE(ReferenceSum, SumBitmapReader<int64_t>)->Apply(BenchmarkSetArgs);
+BENCHMARK_TEMPLATE(ReferenceSum, SumBitmapVectorizeUnroll<int64_t>)
+    ->Apply(BenchmarkSetArgs);
+#endif  // ARROW_WITH_BENCHMARKS_REFERENCE
 
-static void RegressionSumKernel(benchmark::State& state) {
+static void SumKernel(benchmark::State& state) {
   const int64_t array_size = state.range(0) / sizeof(int64_t);
   const double null_percent = static_cast<double>(state.range(1)) / 100.0;
   auto rand = random::RandomArrayGenerator(1923);
@@ -322,7 +326,7 @@ static void RegressionSumKernel(benchmark::State& state) {
   state.SetBytesProcessed(state.iterations() * array_size * sizeof(int64_t));
 }
 
-BENCHMARK(RegressionSumKernel)->Apply(RegressionSetArgs);
+BENCHMARK(SumKernel)->Apply(RegressionSetArgs);
 
 }  // namespace compute
 }  // namespace arrow

cpp/src/arrow/compute/kernels/filter-benchmark.cc

@@ -29,7 +29,7 @@
 namespace arrow {
 namespace compute {
 
-static void RegressionCompareArrayScalarKernel(benchmark::State& state) {
+static void CompareArrayScalarKernel(benchmark::State& state) {
   const int64_t memory_size = state.range(0) / 4;
   const int64_t array_size = memory_size / sizeof(int64_t);
   const double null_percent = static_cast<double>(state.range(1)) / 100.0;
@@ -51,7 +51,7 @@ static void RegressionCompareArrayScalarKernel(benchmark::State& state) {
   state.SetBytesProcessed(state.iterations() * array_size * sizeof(int64_t));
 }
 
-BENCHMARK(RegressionCompareArrayScalarKernel)->Apply(RegressionSetArgs);
+BENCHMARK(CompareArrayScalarKernel)->Apply(RegressionSetArgs);
 
 }  // namespace compute
 }  // namespace arrow

cpp/src/arrow/csv/converter-benchmark.cc

@@ -76,7 +76,7 @@ static void BenchmarkConversion(benchmark::State& state,  // NOLINT non-const re
 
 constexpr size_t num_rows = 10000;
 
-static void RegressionInt64Conversion(
+static void Int64Conversion(
     benchmark::State& state) {  // NOLINT non-const reference
   auto parser = BuildInt64Data(num_rows);
   auto options = ConvertOptions::Defaults();
@@ -84,7 +84,7 @@ static void RegressionInt64Conversion(
   BenchmarkConversion(state, *parser, int64(), options);
 }
 
-static void RegressionFloatConversion(
+static void FloatConversion(
     benchmark::State& state) {  // NOLINT non-const reference
   auto parser = BuildFloatData(num_rows);
   auto options = ConvertOptions::Defaults();
@@ -92,8 +92,8 @@ static void RegressionFloatConversion(
   BenchmarkConversion(state, *parser, float64(), options);
 }
 
-BENCHMARK(RegressionInt64Conversion);
-BENCHMARK(RegressionFloatConversion);
+BENCHMARK(Int64Conversion);
+BENCHMARK(FloatConversion);
 
 }  // namespace csv
 }  // namespace arrow

cpp/src/arrow/csv/parser-benchmark.cc

@@ -56,7 +56,7 @@ static void BenchmarkCSVChunking(benchmark::State& state,  // NOLINT non-const r
   state.SetBytesProcessed(state.iterations() * csv_size);
 }
 
-static void RegressionChunkCSVQuotedBlock(
+static void ChunkCSVQuotedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
   auto csv = BuildCsvData(one_row, num_rows);
   auto options = ParseOptions::Defaults();
@@ -67,7 +67,7 @@ static void RegressionChunkCSVQuotedBlock(
   BenchmarkCSVChunking(state, csv, options);
 }
 
-static void RegressionChunkCSVEscapedBlock(
+static void ChunkCSVEscapedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
   auto csv = BuildCsvData(one_row_escaped, num_rows);
   auto options = ParseOptions::Defaults();
@@ -78,7 +78,7 @@ static void RegressionChunkCSVEscapedBlock(
   BenchmarkCSVChunking(state, csv, options);
 }
 
-static void RegressionChunkCSVNoNewlinesBlock(
+static void ChunkCSVNoNewlinesBlock(
     benchmark::State& state) {  // NOLINT non-const reference
   auto csv = BuildCsvData(one_row_escaped, num_rows);
   auto options = ParseOptions::Defaults();
@@ -121,7 +121,7 @@ static void BenchmarkCSVParsing(benchmark::State& state,  // NOLINT non-const re
   state.SetBytesProcessed(state.iterations() * csv_size);
 }
 
-static void RegressionParseCSVQuotedBlock(
+static void ParseCSVQuotedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
   auto csv = BuildCsvData(one_row, num_rows);
   auto options = ParseOptions::Defaults();
@@ -131,7 +131,7 @@ static void RegressionParseCSVQuotedBlock(
   BenchmarkCSVParsing(state, csv, num_rows, options);
 }
 
-static void RegressionParseCSVEscapedBlock(
+static void ParseCSVEscapedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
   auto csv = BuildCsvData(one_row_escaped, num_rows);
   auto options = ParseOptions::Defaults();
@@ -141,11 +141,11 @@ static void RegressionParseCSVEscapedBlock(
   BenchmarkCSVParsing(state, csv, num_rows, options);
 }
 
-BENCHMARK(RegressionChunkCSVQuotedBlock);
-BENCHMARK(RegressionChunkCSVEscapedBlock);
-BENCHMARK(RegressionChunkCSVNoNewlinesBlock);
-BENCHMARK(RegressionParseCSVQuotedBlock);
-BENCHMARK(RegressionParseCSVEscapedBlock);
+BENCHMARK(ChunkCSVQuotedBlock);
+BENCHMARK(ChunkCSVEscapedBlock);
+BENCHMARK(ChunkCSVNoNewlinesBlock);
+BENCHMARK(ParseCSVQuotedBlock);
+BENCHMARK(ParseCSVEscapedBlock);
 
 }  // namespace csv
 }  // namespace arrow

cpp/src/arrow/gpu/cuda-benchmark.cc

@@ -85,14 +85,12 @@ static void BM_Writer_Unbuffered(benchmark::State& state) {
 BENCHMARK(BM_Writer_Buffered)
     ->RangeMultiplier(16)
     ->Range(1 << 8, 1 << 16)
-    ->MinTime(1.0)
     ->UseRealTime();
 
 BENCHMARK(BM_Writer_Unbuffered)
     ->RangeMultiplier(4)
     ->RangeMultiplier(16)
     ->Range(1 << 8, 1 << 16)
-    ->MinTime(1.0)
     ->UseRealTime();
 
 }  // namespace cuda

cpp/src/arrow/io/file-benchmark.cc

@@ -224,31 +224,13 @@ static void BM_BufferedOutputStreamLargeWritesToPipe(
 // We use real time as we don't want to count CPU time spent in the
 // BackgroundReader thread
 
-BENCHMARK(BM_FileOutputStreamSmallWritesToNull)
-    ->Repetitions(2)
-    ->MinTime(1.0)
-    ->UseRealTime();
-BENCHMARK(BM_FileOutputStreamSmallWritesToPipe)
-    ->Repetitions(2)
-    ->MinTime(1.0)
-    ->UseRealTime();
-BENCHMARK(BM_FileOutputStreamLargeWritesToPipe)
-    ->Repetitions(2)
-    ->MinTime(1.0)
-    ->UseRealTime();
+BENCHMARK(BM_FileOutputStreamSmallWritesToNull)->UseRealTime();
+BENCHMARK(BM_FileOutputStreamSmallWritesToPipe)->UseRealTime();
+BENCHMARK(BM_FileOutputStreamLargeWritesToPipe)->UseRealTime();
 
-BENCHMARK(BM_BufferedOutputStreamSmallWritesToNull)
-    ->Repetitions(2)
-    ->MinTime(1.0)
-    ->UseRealTime();
-BENCHMARK(BM_BufferedOutputStreamSmallWritesToPipe)
-    ->Repetitions(2)
-    ->MinTime(1.0)
-    ->UseRealTime();
-BENCHMARK(BM_BufferedOutputStreamLargeWritesToPipe)
-    ->Repetitions(2)
-    ->MinTime(1.0)
-    ->UseRealTime();
+BENCHMARK(BM_BufferedOutputStreamSmallWritesToNull)->UseRealTime();
+BENCHMARK(BM_BufferedOutputStreamSmallWritesToPipe)->UseRealTime();
+BENCHMARK(BM_BufferedOutputStreamLargeWritesToPipe)->UseRealTime();
 
 #endif  // ifndef _WIN32
 

cpp/src/arrow/io/memory-benchmark.cc

@@ -44,6 +44,7 @@ static const int64_t kL3Size = cpu_info->CacheSize(CpuInfo::L3_CACHE);
 constexpr size_t kMemoryPerCore = 32 * 1024 * 1024;
 using BufferPtr = std::shared_ptr<Buffer>;
 
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
 #ifndef _MSC_VER
 
 #ifdef ARROW_AVX512
@@ -200,6 +201,7 @@ BENCHMARK_TEMPLATE(MemoryBandwidth, StreamReadWrite)->Apply(SetMemoryBandwidthAr
 BENCHMARK_TEMPLATE(MemoryBandwidth, PlatformMemcpy)->Apply(SetMemoryBandwidthArgs);
 
 #endif  // _MSC_VER
+#endif  // ARROW_WITH_BENCHMARKS_REFERENCE
 
 static void ParallelMemoryCopy(benchmark::State& state) {  // NOLINT non-const reference
   const int64_t n_threads = state.range(0);

cpp/src/arrow/ipc/read-write-benchmark.cc

@@ -99,16 +99,7 @@ static void BM_ReadRecordBatch(benchmark::State& state) {  // NOLINT non-const r
   state.SetBytesProcessed(int64_t(state.iterations()) * kTotalSize);
 }
 
-BENCHMARK(BM_WriteRecordBatch)
-    ->RangeMultiplier(4)
-    ->Range(1, 1 << 13)
-    ->MinTime(1.0)
-    ->UseRealTime();
-
-BENCHMARK(BM_ReadRecordBatch)
-    ->RangeMultiplier(4)
-    ->Range(1, 1 << 13)
-    ->MinTime(1.0)
-    ->UseRealTime();
+BENCHMARK(BM_WriteRecordBatch)->RangeMultiplier(4)->Range(1, 1 << 13)->UseRealTime();
+BENCHMARK(BM_ReadRecordBatch)->RangeMultiplier(4)->Range(1, 1 << 13)->UseRealTime();
 
 }  // namespace arrow

cpp/src/arrow/json/parser-benchmark.cc

@@ -63,7 +63,7 @@ static void BenchmarkJSONChunking(benchmark::State& state,
   state.SetBytesProcessed(state.iterations() * json->size());
 }
 
-static void RegressionChunkJSONPrettyPrinted(
+static void ChunkJSONPrettyPrinted(
     benchmark::State& state) {  // NOLINT non-const reference
   const int32_t num_rows = 5000;
 
@@ -75,7 +75,7 @@ static void RegressionChunkJSONPrettyPrinted(
   BenchmarkJSONChunking(state, std::make_shared<Buffer>(json), options);
 }
 
-static void RegressionChunkJSONLineDelimited(
+static void ChunkJSONLineDelimited(
     benchmark::State& state) {  // NOLINT non-const reference
   const int32_t num_rows = 5000;
 
@@ -102,7 +102,7 @@ static void BenchmarkJSONParsing(benchmark::State& state,  // NOLINT non-const r
   state.SetBytesProcessed(state.iterations() * json->size());
 }
 
-static void RegressionParseJSONBlockWithSchema(
+static void ParseJSONBlockWithSchema(
     benchmark::State& state) {  // NOLINT non-const reference
   const int32_t num_rows = 5000;
   auto options = ParseOptions::Defaults();
@@ -145,7 +145,7 @@ static void BenchmarkReadJSONBlockWithSchema(
   BenchmarkJSONReading(state, json, num_rows, read_options, parse_options);
 }
 
-static void RegressionReadJSONBlockWithSchemaSingleThread(
+static void ReadJSONBlockWithSchemaSingleThread(
     benchmark::State& state) {  // NOLINT non-const reference
   BenchmarkReadJSONBlockWithSchema(state, false);
 }
@@ -155,11 +155,11 @@ static void ReferenceReadJSONBlockWithSchemaMultiThread(
   BenchmarkReadJSONBlockWithSchema(state, true);
 }
 
-BENCHMARK(RegressionChunkJSONPrettyPrinted);
-BENCHMARK(RegressionChunkJSONLineDelimited);
-BENCHMARK(RegressionParseJSONBlockWithSchema);
+BENCHMARK(ChunkJSONPrettyPrinted);
+BENCHMARK(ChunkJSONLineDelimited);
+BENCHMARK(ParseJSONBlockWithSchema);
 
-BENCHMARK(RegressionReadJSONBlockWithSchemaSingleThread);
+BENCHMARK(ReadJSONBlockWithSchemaSingleThread);
 BENCHMARK(ReferenceReadJSONBlockWithSchemaMultiThread)->UseRealTime();
 
 }  // namespace json

cpp/src/arrow/util/bit-util-benchmark.cc

@@ -28,8 +28,6 @@
 
 namespace arrow {
 
-using internal::CopyBitmap;
-
 namespace BitUtil {
 
 // A naive bitmap reader implementation, meant as a baseline against
@@ -180,7 +178,7 @@ static void ReferenceNaiveBitmapReader(benchmark::State& state) {
   BenchmarkBitmapReader<NaiveBitmapReader>(state, state.range(0));
 }
 
-static void RegressionBitmapReader(benchmark::State& state) {
+static void BitmapReader(benchmark::State& state) {
   BenchmarkBitmapReader<internal::BitmapReader>(state, state.range(0));
 }
 
@@ -188,11 +186,11 @@ static void ReferenceNaiveBitmapWriter(benchmark::State& state) {
   BenchmarkBitmapWriter<NaiveBitmapWriter>(state, state.range(0));
 }
 
-static void RegressionBitmapWriter(benchmark::State& state) {
+static void BitmapWriter(benchmark::State& state) {
   BenchmarkBitmapWriter<internal::BitmapWriter>(state, state.range(0));
 }
 
-static void RegressionFirstTimeBitmapWriter(benchmark::State& state) {
+static void FirstTimeBitmapWriter(benchmark::State& state) {
   BenchmarkBitmapWriter<internal::FirstTimeBitmapWriter>(state, state.range(0));
 }
 
@@ -210,18 +208,18 @@ struct GenerateBitsUnrolledFunctor {
   }
 };
 
-static void RegressionGenerateBits(benchmark::State& state) {
+static void GenerateBits(benchmark::State& state) {
   BenchmarkGenerateBits<GenerateBitsFunctor>(state, state.range(0));
 }
 
-static void RegressionGenerateBitsUnrolled(benchmark::State& state) {
+static void GenerateBitsUnrolled(benchmark::State& state) {
   BenchmarkGenerateBits<GenerateBitsUnrolledFunctor>(state, state.range(0));
 }
 
 constexpr int64_t kBufferSize = 1024 * 8;
 
 template <int64_t Offset = 0>
-static void RegressionCopyBitmap(benchmark::State& state) {  // NOLINT non-const reference
+static void CopyBitmap(benchmark::State& state) {  // NOLINT non-const reference
   const int64_t buffer_size = state.range(0);
   const int64_t bits_size = buffer_size * 8;
   std::shared_ptr<Buffer> buffer = CreateRandomBuffer(buffer_size);
@@ -235,34 +233,33 @@ static void RegressionCopyBitmap(benchmark::State& state) {  // NOLINT non-const
   ABORT_NOT_OK(AllocateEmptyBitmap(pool, length, &copy));
 
   for (auto _ : state) {
-    CopyBitmap(src, offset, length, copy->mutable_data(), 0, false);
+    internal::CopyBitmap(src, offset, length, copy->mutable_data(), 0, false);
   }
 
   state.SetBytesProcessed(state.iterations() * buffer_size);
 }
 
-static void RegressionCopyBitmapWithoutOffset(
+static void CopyBitmapWithoutOffset(
     benchmark::State& state) {  // NOLINT non-const reference
-  RegressionCopyBitmap<0>(state);
+  CopyBitmap<0>(state);
 }
 
 // Trigger the slow path where the buffer is not byte aligned.
-static void RegressionCopyBitmapWithOffset(
-    benchmark::State& state) {  // NOLINT non-const reference
-  RegressionCopyBitmap<4>(state);
+static void CopyBitmapWithOffset(benchmark::State& state) {  // NOLINT non-const reference
+  CopyBitmap<4>(state);
 }
 
-BENCHMARK(RegressionCopyBitmapWithoutOffset)->Arg(kBufferSize);
-BENCHMARK(RegressionCopyBitmapWithOffset)->Arg(kBufferSize);
+BENCHMARK(CopyBitmapWithoutOffset)->Arg(kBufferSize);
+BENCHMARK(CopyBitmapWithOffset)->Arg(kBufferSize);
 
 BENCHMARK(ReferenceNaiveBitmapReader)->Arg(kBufferSize);
-BENCHMARK(RegressionBitmapReader)->Arg(kBufferSize);
+BENCHMARK(BitmapReader)->Arg(kBufferSize);
 BENCHMARK(ReferenceNaiveBitmapWriter)->Arg(kBufferSize);
-BENCHMARK(RegressionBitmapWriter)->Arg(kBufferSize);
+BENCHMARK(BitmapWriter)->Arg(kBufferSize);
 
-BENCHMARK(RegressionFirstTimeBitmapWriter)->Arg(kBufferSize);
-BENCHMARK(RegressionGenerateBits)->Arg(kBufferSize);
-BENCHMARK(RegressionGenerateBitsUnrolled)->Arg(kBufferSize);
+BENCHMARK(FirstTimeBitmapWriter)->Arg(kBufferSize);
+BENCHMARK(GenerateBits)->Arg(kBufferSize);
+BENCHMARK(GenerateBitsUnrolled)->Arg(kBufferSize);
 
 }  // namespace BitUtil
 }  // namespace arrow

cpp/src/arrow/util/compression-benchmark.cc

@@ -180,31 +180,15 @@ static void BM_StreamingDecompression(
   BM_StreamingDecompression(COMPRESSION, data, state);
 }
 
-BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::GZIP)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
-BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::BROTLI)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
-BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::ZSTD)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
-BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::LZ4)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
+BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::GZIP);
+BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::BROTLI);
+BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::ZSTD);
+BENCHMARK_TEMPLATE(BM_StreamingCompression, Compression::LZ4);
 
-BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::GZIP)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
-BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::BROTLI)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
-BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::ZSTD)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
-BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::LZ4)
-    ->Unit(benchmark::kMillisecond)
-    ->Repetitions(1);
+BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::GZIP);
+BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::BROTLI);
+BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::ZSTD);
+BENCHMARK_TEMPLATE(BM_StreamingDecompression, Compression::LZ4);
 
 }  // namespace util
 }  // namespace arrow

cpp/src/arrow/util/decimal-benchmark.cc

@@ -39,7 +39,7 @@ static void BM_FromString(benchmark::State& state) {  // NOLINT non-const refere
   state.SetItemsProcessed(state.iterations() * values.size());
 }
 
-BENCHMARK(BM_FromString)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_FromString);
 
 }  // namespace Decimal
 }  // namespace arrow

cpp/src/arrow/util/hashing-benchmark.cc

@@ -114,15 +114,10 @@ static void BM_HashLargeStrings(benchmark::State& state) {  // NOLINT non-const
 // ----------------------------------------------------------------------
 // Benchmark declarations
 
-static constexpr int32_t kRepetitions = 1;
-
-BENCHMARK(BM_HashIntegers)->Repetitions(kRepetitions)->Unit(benchmark::kMicrosecond);
-
-BENCHMARK(BM_HashSmallStrings)->Repetitions(kRepetitions)->Unit(benchmark::kMicrosecond);
-
-BENCHMARK(BM_HashMediumStrings)->Repetitions(kRepetitions)->Unit(benchmark::kMicrosecond);
-
-BENCHMARK(BM_HashLargeStrings)->Repetitions(kRepetitions)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_HashIntegers);
+BENCHMARK(BM_HashSmallStrings);
+BENCHMARK(BM_HashMediumStrings);
+BENCHMARK(BM_HashLargeStrings);
 
 }  // namespace internal
 }  // namespace arrow

cpp/src/arrow/util/int-util-benchmark.cc

@@ -97,13 +97,10 @@ static void BM_DetectIntWidthNulls(
   state.SetBytesProcessed(state.iterations() * values.size() * sizeof(uint64_t));
 }
 
-BENCHMARK(BM_DetectUIntWidthNoNulls)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
-
-BENCHMARK(BM_DetectUIntWidthNulls)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
-
-BENCHMARK(BM_DetectIntWidthNoNulls)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
-
-BENCHMARK(BM_DetectIntWidthNulls)->MinTime(1.0)->Unit(benchmark::kMicrosecond);
+BENCHMARK(BM_DetectUIntWidthNoNulls);
+BENCHMARK(BM_DetectUIntWidthNulls);
+BENCHMARK(BM_DetectIntWidthNoNulls);
+BENCHMARK(BM_DetectIntWidthNulls);
 
 }  // namespace internal
 }  // namespace arrow

cpp/src/arrow/util/lazy-benchmark.cc

@@ -27,6 +27,8 @@
 
 namespace arrow {
 
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
 static constexpr int64_t kSize = 100000000;
 
 template <typename T = int32_t>
@@ -46,7 +48,7 @@ void BM_for_loop(benchmark::State& state) {
   }
 }
 
-BENCHMARK(BM_for_loop)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_for_loop);
 
 // For comparison: pure copy without any changes
 void BM_std_copy(benchmark::State& state) {
@@ -58,7 +60,7 @@ void BM_std_copy(benchmark::State& state) {
   }
 }
 
-BENCHMARK(BM_std_copy)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_std_copy);
 
 // For comparison: pure copy with type convesion.
 void BM_std_copy_converting(benchmark::State& state) {
@@ -71,7 +73,7 @@ void BM_std_copy_converting(benchmark::State& state) {
   }
 }
 
-BENCHMARK(BM_std_copy_converting)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_std_copy_converting);
 
 // std::copy with a lazy range as a source
 void BM_lazy_copy(benchmark::State& state) {
@@ -85,7 +87,7 @@ void BM_lazy_copy(benchmark::State& state) {
   }
 }
 
-BENCHMARK(BM_lazy_copy)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_lazy_copy);
 
 // std::copy with a lazy range which does static cast.
 // Should be the same performance as std::copy with differtly typed iterators
@@ -101,7 +103,7 @@ void BM_lazy_copy_converting(benchmark::State& state) {
   }
 }
 
-BENCHMARK(BM_lazy_copy_converting)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_lazy_copy_converting);
 
 // For loop with a post-increment of a lazy operator
 void BM_lazy_postinc(benchmark::State& state) {
@@ -119,6 +121,8 @@ void BM_lazy_postinc(benchmark::State& state) {
   }
 }
 
-BENCHMARK(BM_lazy_postinc)->Repetitions(3)->Unit(benchmark::kMillisecond);
+BENCHMARK(BM_lazy_postinc);
+
+#endif  // ARROW_WITH_BENCHMARKS_REFERENCE
 
 }  // namespace arrow

cpp/src/arrow/util/machine-benchmark.cc

@@ -28,6 +28,8 @@
 
 namespace arrow {
 
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
 // Generate a vector of indices such as following the indices describes
 // a path over the whole vector.  The path is randomized to avoid triggering
 // automatic prefetching in the CPU.
@@ -65,6 +67,8 @@ static void BM_memory_latency(benchmark::State& state) {
   state.SetItemsProcessed(state.iterations());
 }
 
-BENCHMARK(BM_memory_latency)->RangeMultiplier(2)->Range(2 << 10, 2 << 24);
+BENCHMARK(BM_memory_latency)->Repetitions(1)->RangeMultiplier(2)->Range(2 << 10, 2 << 24);
+
+#endif  // ARROW_WITH_BENCHMARKS_REFERENCE
 
 }  // namespace arrow

cpp/src/arrow/util/thread-pool-benchmark.cc

@@ -168,6 +168,8 @@ static void WorkloadCost_Customize(benchmark::internal::Benchmark* b) {
     b->Args({w});
   }
   b->ArgNames({"task_cost"});
+  b->UseRealTime();
+  b->Repetitions(1);
 }
 
 static void ThreadPoolSpawn_Customize(benchmark::internal::Benchmark* b) {
@@ -177,26 +179,14 @@ static void ThreadPoolSpawn_Customize(benchmark::internal::Benchmark* b) {
     }
   }
   b->ArgNames({"threads", "task_cost"});
+  b->UseRealTime();
+  b->Repetitions(1);
 }
 
-static const int kRepetitions = 1;
-
-BENCHMARK(BM_WorkloadCost)->Repetitions(kRepetitions)->Apply(WorkloadCost_Customize);
-
-BENCHMARK(BM_ThreadPoolSpawn)
-    ->UseRealTime()
-    ->Repetitions(kRepetitions)
-    ->Apply(ThreadPoolSpawn_Customize);
-
-BENCHMARK(BM_SerialTaskGroup)
-    ->UseRealTime()
-    ->Repetitions(kRepetitions)
-    ->Apply(WorkloadCost_Customize);
-
-BENCHMARK(BM_ThreadedTaskGroup)
-    ->UseRealTime()
-    ->Repetitions(kRepetitions)
-    ->Apply(ThreadPoolSpawn_Customize);
+BENCHMARK(BM_WorkloadCost)->Apply(WorkloadCost_Customize);
+BENCHMARK(BM_SerialTaskGroup)->Apply(WorkloadCost_Customize);
+BENCHMARK(BM_ThreadPoolSpawn)->Apply(ThreadPoolSpawn_Customize);
+BENCHMARK(BM_ThreadedTaskGroup)->Apply(ThreadPoolSpawn_Customize);
 
 }  // namespace internal
 }  // namespace arrow

cpp/src/arrow/util/trie-benchmark.cc

@@ -210,12 +210,10 @@ static void BM_InlinedTrieLookupNotFound(
   BenchmarkInlinedTrieLookups(state, {"None", "1.0", "", "abc"});
 }
 
-static const int kRepetitions = 2;
-
-BENCHMARK(BM_TrieLookupFound)->Repetitions(kRepetitions);
-BENCHMARK(BM_TrieLookupNotFound)->Repetitions(kRepetitions);
-BENCHMARK(BM_InlinedTrieLookupFound)->Repetitions(kRepetitions);
-BENCHMARK(BM_InlinedTrieLookupNotFound)->Repetitions(kRepetitions);
+BENCHMARK(BM_TrieLookupFound);
+BENCHMARK(BM_TrieLookupNotFound);
+BENCHMARK(BM_InlinedTrieLookupFound);
+BENCHMARK(BM_InlinedTrieLookupNotFound);
 
 }  // namespace internal
 }  // namespace arrow

cpp/src/arrow/util/utf8-util-benchmark.cc

@@ -81,8 +81,7 @@ static void BM_ValidateTinyNonAscii(
 
 static void BM_ValidateSmallAscii(
     benchmark::State& state) {  // NOLINT non-const reference
-  BenchmarkUTF8Validation(state, valid_ascii, true);
-}
+  BenchmarkUTF8Validation(state, valid_ascii, true); }
 
 static void BM_ValidateSmallAlmostAscii(
     benchmark::State& state) {  // NOLINT non-const reference
@@ -112,16 +111,14 @@ static void BM_ValidateLargeNonAscii(
   BenchmarkUTF8Validation(state, s, true);
 }
 
-static const int kRepetitions = 1;
-
-BENCHMARK(BM_ValidateTinyAscii)->Repetitions(kRepetitions);
-BENCHMARK(BM_ValidateTinyNonAscii)->Repetitions(kRepetitions);
-BENCHMARK(BM_ValidateSmallAscii)->Repetitions(kRepetitions);
-BENCHMARK(BM_ValidateSmallAlmostAscii)->Repetitions(kRepetitions);
-BENCHMARK(BM_ValidateSmallNonAscii)->Repetitions(kRepetitions);
-BENCHMARK(BM_ValidateLargeAscii)->Repetitions(kRepetitions);
-BENCHMARK(BM_ValidateLargeAlmostAscii)->Repetitions(kRepetitions);
-BENCHMARK(BM_ValidateLargeNonAscii)->Repetitions(kRepetitions);
+BENCHMARK(BM_ValidateTinyAscii);
+BENCHMARK(BM_ValidateTinyNonAscii);
+BENCHMARK(BM_ValidateSmallAscii);
+BENCHMARK(BM_ValidateSmallAlmostAscii);
+BENCHMARK(BM_ValidateSmallNonAscii);
+BENCHMARK(BM_ValidateLargeAscii);
+BENCHMARK(BM_ValidateLargeAlmostAscii);
+BENCHMARK(BM_ValidateLargeNonAscii);
 
 }  // namespace util
 }  // namespace arrow

dev/archery/archery/cli.py

@@ -214,7 +214,7 @@ def benchmark_list(ctx, src, preserve, output, cmake_extras, baseline):
 @click.option("--suite-filter", metavar="<regex>", show_default=True,
               type=str, default=None, help="Regex filtering benchmark suites.")
 @click.option("--benchmark-filter", metavar="<regex>", show_default=True,
-              type=str, default=DEFAULT_BENCHMARK_FILTER,
+              type=str, default=None,
               help="Regex filtering benchmark suites.")
 @click.option("--preserve", type=bool, default=False, show_default=True,
               is_flag=True, help="Preserve workspace for investigation.")