Refactor csv benchmarks

cpp/src/arrow/compute/benchmark-util.h

@@ -32,40 +32,42 @@ static const int64_t kL1Size = cpu_info->CacheSize(CpuInfo::L1_CACHE);
 static const int64_t kL2Size = cpu_info->CacheSize(CpuInfo::L2_CACHE);
 static const int64_t kL3Size = cpu_info->CacheSize(CpuInfo::L3_CACHE);
 static const int64_t kCantFitInL3Size = kL3Size * 4;
+static const std::vector<int64_t> kMemorySizes = {kL1Size, kL2Size, kL3Size,
+                                                  kCantFitInL3Size};
 
 template <typename Func>
 struct BenchmarkArgsType;
 
+// Pattern matching that extracts the vector element type of Benchmark::Args()
 template <typename Values>
 struct BenchmarkArgsType<benchmark::internal::Benchmark* (
     benchmark::internal::Benchmark::*)(const std::vector<Values>&)> {
   using type = Values;
 };
 
-void BenchmarkSetArgs(benchmark::internal::Benchmark* bench) {
-  // Benchmark changed its parameter type between releases from
-  // int to int64_t. As it doesn't have version macros, we need
-  // to apply C++ template magic.
-  using ArgsType =
-      typename BenchmarkArgsType<decltype(&benchmark::internal::Benchmark::Args)>::type;
+// Benchmark changed its parameter type between releases from
+// int to int64_t. As it doesn't have version macros, we need
+// to apply C++ template magic.
+using ArgsType =
+    typename BenchmarkArgsType<decltype(&benchmark::internal::Benchmark::Args)>::type;
+
+void BenchmarkSetArgsWithSizes(benchmark::internal::Benchmark* bench,
+                               const std::vector<int64_t>& sizes = kMemorySizes) {
   bench->Unit(benchmark::kMicrosecond);
 
-  for (auto size : {kL1Size, kL2Size, kL3Size, kCantFitInL3Size})
+  for (auto size : sizes)
     for (auto nulls : std::vector<ArgsType>({0, 1, 10, 50}))
       bench->Args({static_cast<ArgsType>(size), nulls});
 }
 
-void RegressionSetArgs(benchmark::internal::Benchmark* bench) {
-  // Benchmark changed its parameter type between releases from
-  // int to int64_t. As it doesn't have version macros, we need
-  // to apply C++ template magic.
-  using ArgsType =
-      typename BenchmarkArgsType<decltype(&benchmark::internal::Benchmark::Args)>::type;
-  bench->Unit(benchmark::kMicrosecond);
+void BenchmarkSetArgs(benchmark::internal::Benchmark* bench) {
+  BenchmarkSetArgsWithSizes(bench, kMemorySizes);
+}
 
-  // Regressions should only bench L1 data for better stability
-  for (auto nulls : std::vector<ArgsType>({0, 1, 10, 50}))
-    bench->Args({static_cast<ArgsType>(kL1Size), nulls});
+void RegressionSetArgs(benchmark::internal::Benchmark* bench) {
+  // Regression do not need to account for cache hierarchy, thus optimize for
+  // the best case.
+  BenchmarkSetArgsWithSizes(bench, {kL1Size});
 }
 
 }  // namespace compute

cpp/src/arrow/csv/converter-benchmark.cc

@@ -74,24 +74,26 @@ static void BenchmarkConversion(benchmark::State& state,  // NOLINT non-const re
   state.SetItemsProcessed(state.iterations() * parser.num_rows());
 }
 
-static void BM_Int64Conversion(benchmark::State& state) {  // NOLINT non-const reference
-  const int32_t num_rows = 10000;
+constexpr size_t num_rows = 10000;
+
+static void RegressionInt64Conversion(
+    benchmark::State& state) {  // NOLINT non-const reference
   auto parser = BuildInt64Data(num_rows);
   auto options = ConvertOptions::Defaults();
 
   BenchmarkConversion(state, *parser, int64(), options);
 }
 
-static void BM_FloatConversion(benchmark::State& state) {  // NOLINT non-const reference
-  const int32_t num_rows = 10000;
+static void RegressionFloatConversion(
+    benchmark::State& state) {  // NOLINT non-const reference
   auto parser = BuildFloatData(num_rows);
   auto options = ConvertOptions::Defaults();
 
   BenchmarkConversion(state, *parser, float64(), options);
 }
 
-BENCHMARK(BM_Int64Conversion)->Repetitions(3);
-BENCHMARK(BM_FloatConversion)->Repetitions(3);
+BENCHMARK(RegressionInt64Conversion);
+BENCHMARK(RegressionFloatConversion);
 
 }  // namespace csv
 }  // namespace arrow

cpp/src/arrow/csv/parser-benchmark.cc

@@ -28,20 +28,16 @@
 namespace arrow {
 namespace csv {
 
-static std::string BuildQuotedData(int32_t num_rows = 10000) {
-  std::string one_row = "abc,\"d,f\",12.34,\n";
-  std::stringstream ss;
-  for (int32_t i = 0; i < num_rows; ++i) {
-    ss << one_row;
-  }
-  return ss.str();
-}
+// Can't have static str.
+const char* one_row = "abc,\"d,f\",12.34,\n";
+const char* one_row_escaped = "abc,d\\,f,12.34,\n";
 
-static std::string BuildEscapedData(int32_t num_rows = 10000) {
-  std::string one_row = "abc,d\\,f,12.34,\n";
+size_t num_rows = (1024 * 8) / strlen(one_row);
+
+static std::string BuildCsvData(const std::string& row, size_t repeat) {
   std::stringstream ss;
-  for (int32_t i = 0; i < num_rows; ++i) {
-    ss << one_row;
+  for (size_t i = 0; i < repeat; ++i) {
+    ss << row;
   }
   return ss.str();
 }
@@ -49,23 +45,20 @@ static std::string BuildEscapedData(int32_t num_rows = 10000) {
 static void BenchmarkCSVChunking(benchmark::State& state,  // NOLINT non-const reference
                                  const std::string& csv, ParseOptions options) {
   Chunker chunker(options);
+  const uint32_t csv_size = static_cast<uint32_t>(csv.size());
 
   while (state.KeepRunning()) {
-    uint32_t chunk_size;
-    ABORT_NOT_OK(
-        chunker.Process(csv.data(), static_cast<uint32_t>(csv.size()), &chunk_size));
-    if (chunk_size != csv.size()) {
-      std::cerr << "Parsing incomplete\n";
-      std::abort();
-    }
+    uint32_t chunk_size = 0;
+    ABORT_NOT_OK(chunker.Process(csv.data(), csv_size, &chunk_size));
+    benchmark::DoNotOptimize(chunk_size);
   }
-  state.SetBytesProcessed(state.iterations() * csv.size());
+
+  state.SetBytesProcessed(state.iterations() * csv_size);
 }
 
-static void BM_ChunkCSVQuotedBlock(
+static void RegressionChunkCSVQuotedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
-  const int32_t num_rows = 5000;
-  auto csv = BuildQuotedData(num_rows);
+  auto csv = BuildCsvData(one_row, num_rows);
   auto options = ParseOptions::Defaults();
   options.quoting = true;
   options.escaping = false;
@@ -74,10 +67,9 @@ static void BM_ChunkCSVQuotedBlock(
   BenchmarkCSVChunking(state, csv, options);
 }
 
-static void BM_ChunkCSVEscapedBlock(
+static void RegressionChunkCSVEscapedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
-  const int32_t num_rows = 5000;
-  auto csv = BuildEscapedData(num_rows);
+  auto csv = BuildCsvData(one_row_escaped, num_rows);
   auto options = ParseOptions::Defaults();
   options.quoting = false;
   options.escaping = true;
@@ -86,31 +78,30 @@ static void BM_ChunkCSVEscapedBlock(
   BenchmarkCSVChunking(state, csv, options);
 }
 
-static void BM_ChunkCSVNoNewlinesBlock(
+static void RegressionChunkCSVNoNewlinesBlock(
     benchmark::State& state) {  // NOLINT non-const reference
-  const int32_t num_rows = 5000;
-  auto csv = BuildEscapedData(num_rows);
+  auto csv = BuildCsvData(one_row_escaped, num_rows);
   auto options = ParseOptions::Defaults();
   options.quoting = true;
   options.escaping = false;
   options.newlines_in_values = false;
 
   BenchmarkCSVChunking(state, csv, options);
+  // Provides better regression stability with timings rather than bogus
+  // bandwidth.
+  state.SetBytesProcessed(0);
 }
 
 static void BenchmarkCSVParsing(benchmark::State& state,  // NOLINT non-const reference
-                                const std::string& csv, int32_t num_rows,
+                                const std::string& csv, int32_t rows,
                                 ParseOptions options) {
-  BlockParser parser(options, -1, num_rows + 1);
+  BlockParser parser(options, -1, rows + 1);
+  const uint32_t csv_size = static_cast<uint32_t>(csv.size());
 
   while (state.KeepRunning()) {
-    uint32_t parsed_size;
-    ABORT_NOT_OK(
-        parser.Parse(csv.data(), static_cast<uint32_t>(csv.size()), &parsed_size));
-    if (parsed_size != csv.size() || parser.num_rows() != num_rows) {
-      std::cerr << "Parsing incomplete\n";
-      std::abort();
-    }
+    uint32_t parsed_size = 0;
+    ABORT_NOT_OK(parser.Parse(csv.data(), csv_size, &parsed_size));
+
     // Include performance of visiting the parsed values, as that might
     // vary depending on the parser's internal data structures.
     bool dummy_quoted = false;
@@ -126,13 +117,13 @@ static void BenchmarkCSVParsing(benchmark::State& state,  // NOLINT non-const re
       benchmark::DoNotOptimize(dummy_quoted);
     }
   }
-  state.SetBytesProcessed(state.iterations() * csv.size());
+
+  state.SetBytesProcessed(state.iterations() * csv_size);
 }
 
-static void BM_ParseCSVQuotedBlock(
+static void RegressionParseCSVQuotedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
-  const int32_t num_rows = 5000;
-  auto csv = BuildQuotedData(num_rows);
+  auto csv = BuildCsvData(one_row, num_rows);
   auto options = ParseOptions::Defaults();
   options.quoting = true;
   options.escaping = false;
@@ -140,10 +131,9 @@ static void BM_ParseCSVQuotedBlock(
   BenchmarkCSVParsing(state, csv, num_rows, options);
 }
 
-static void BM_ParseCSVEscapedBlock(
+static void RegressionParseCSVEscapedBlock(
     benchmark::State& state) {  // NOLINT non-const reference
-  const int32_t num_rows = 5000;
-  auto csv = BuildEscapedData(num_rows);
+  auto csv = BuildCsvData(one_row_escaped, num_rows);
   auto options = ParseOptions::Defaults();
   options.quoting = false;
   options.escaping = true;
@@ -151,11 +141,11 @@ static void BM_ParseCSVEscapedBlock(
   BenchmarkCSVParsing(state, csv, num_rows, options);
 }
 
-BENCHMARK(BM_ChunkCSVQuotedBlock)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_ChunkCSVEscapedBlock)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_ChunkCSVNoNewlinesBlock)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_ParseCSVQuotedBlock)->Repetitions(3)->Unit(benchmark::kMicrosecond);
-BENCHMARK(BM_ParseCSVEscapedBlock)->Repetitions(3)->Unit(benchmark::kMicrosecond);
+BENCHMARK(RegressionChunkCSVQuotedBlock);
+BENCHMARK(RegressionChunkCSVEscapedBlock);
+BENCHMARK(RegressionChunkCSVNoNewlinesBlock);
+BENCHMARK(RegressionParseCSVQuotedBlock);
+BENCHMARK(RegressionParseCSVEscapedBlock);
 
 }  // namespace csv
 }  // namespace arrow