address comments

cpp/src/arrow/builder-benchmark.cc

@@ -47,22 +47,22 @@ static VectorType AlmostU8CompressibleVector() {
   return data;
 }
 
-constexpr int64_t kFinalSize = 256;
+constexpr int64_t kRounds = 256;
 static VectorType kData = AlmostU8CompressibleVector();
 constexpr int64_t kBytesProcessPerRound = kNumberOfElements * sizeof(ValueType);
-constexpr int64_t kBytesProcessed = kFinalSize * kBytesProcessPerRound;
+constexpr int64_t kBytesProcessed = kRounds * kBytesProcessPerRound;
 
 static const char* kBinaryString = "12345678";
 static arrow::util::string_view kBinaryView(kBinaryString);
 
 // This benchmarks acts as a reference to the native std::vector
-// implementation. It appends kFinalSize chunks into a vector.
+// implementation. It appends kRounds chunks into a vector.
 static void ReferenceBuildVectorNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
   for (auto _ : state) {
     std::vector<int64_t> builder;
 
-    for (int i = 0; i < kFinalSize; i++) {
+    for (int i = 0; i < kRounds; i++) {
       builder.insert(builder.end(), kData.cbegin(), kData.cend());
     }
   }
@@ -70,12 +70,11 @@ static void ReferenceBuildVectorNoNulls(
   state.SetBytesProcessed(state.iterations() * kBytesProcessed);
 }
 
-static void BuildPrimitiveArrayNoNulls(
-    benchmark::State& state) {  // NOLINT non-const reference
+static void BuildIntArrayNoNulls(benchmark::State& state) {  // NOLINT non-const reference
   for (auto _ : state) {
     Int64Builder builder;
 
-    for (int i = 0; i < kFinalSize; i++) {
+    for (int i = 0; i < kRounds; i++) {
       ABORT_NOT_OK(builder.AppendValues(kData.data(), kData.size(), nullptr));
     }
 
@@ -91,7 +90,7 @@ static void BuildAdaptiveIntNoNulls(
   for (auto _ : state) {
     AdaptiveIntBuilder builder;
 
-    for (int i = 0; i < kFinalSize; i++) {
+    for (int i = 0; i < kRounds; i++) {
       ABORT_NOT_OK(builder.AppendValues(kData.data(), kData.size(), nullptr));
     }
 
@@ -107,7 +106,7 @@ static void BuildAdaptiveIntNoNullsScalarAppend(
   for (auto _ : state) {
     AdaptiveIntBuilder builder;
 
-    for (int i = 0; i < kFinalSize; i++) {
+    for (int i = 0; i < kRounds; i++) {
       for (size_t j = 0; j < kData.size(); j++) {
         ABORT_NOT_OK(builder.Append(kData[i]))
       }
@@ -123,13 +122,13 @@ static void BuildAdaptiveIntNoNullsScalarAppend(
 static void BuildBooleanArrayNoNulls(
     benchmark::State& state) {  // NOLINT non-const reference
 
-  size_t n_bytes = kData.size() * sizeof(ValueType);
+  size_t n_bytes = kBytesProcessPerRound;
   const uint8_t* data = reinterpret_cast<const uint8_t*>(kData.data());
 
   for (auto _ : state) {
     BooleanBuilder builder;
 
-    for (int i = 0; i < kFinalSize; i++) {
+    for (int i = 0; i < kRounds; i++) {
       ABORT_NOT_OK(builder.AppendValues(data, n_bytes));
     }
 
@@ -144,7 +143,7 @@ static void BuildBinaryArray(benchmark::State& state) {  // NOLINT non-const ref
   for (auto _ : state) {
     BinaryBuilder builder;
 
-    for (int64_t i = 0; i < kFinalSize * kNumberOfElements; i++) {
+    for (int64_t i = 0; i < kRounds * kNumberOfElements; i++) {
       ABORT_NOT_OK(builder.Append(kBinaryView));
     }
 
@@ -163,7 +162,7 @@ static void BuildChunkedBinaryArray(
   for (auto _ : state) {
     internal::ChunkedBinaryBuilder builder(kChunkSize);
 
-    for (int64_t i = 0; i < kFinalSize * kNumberOfElements; i++) {
+    for (int64_t i = 0; i < kRounds * kNumberOfElements; i++) {
       ABORT_NOT_OK(builder.Append(kBinaryView));
     }
 
@@ -181,7 +180,7 @@ static void BuildFixedSizeBinaryArray(
   for (auto _ : state) {
     FixedSizeBinaryBuilder builder(type);
 
-    for (int64_t i = 0; i < kFinalSize * kNumberOfElements; i++) {
+    for (int64_t i = 0; i < kRounds * kNumberOfElements; i++) {
       ABORT_NOT_OK(builder.Append(kBinaryView));
     }
 
@@ -292,7 +291,7 @@ static void BenchmarkScalarDictionaryArray(
   for (auto _ : state) {
     DictionaryBuilder<Int64Type> builder(default_memory_pool());
 
-    for (int64_t i = 0; i < kFinalSize; i++) {
+    for (int64_t i = 0; i < kRounds; i++) {
       for (const auto value : fodder) {
         ABORT_NOT_OK(builder.Append(value));
       }
@@ -333,7 +332,7 @@ static void BuildStringDictionaryArray(
   for (auto _ : state) {
     BinaryDictionaryBuilder builder(default_memory_pool());
 
-    for (int64_t i = 0; i < kFinalSize; i++) {
+    for (int64_t i = 0; i < kRounds; i++) {
       for (const auto& value : fodder) {
         ABORT_NOT_OK(builder.Append(value));
       }
@@ -343,7 +342,7 @@ static void BuildStringDictionaryArray(
     ABORT_NOT_OK(builder.Finish(&out));
   }
 
-  state.SetBytesProcessed(state.iterations() * fodder_size * kFinalSize);
+  state.SetBytesProcessed(state.iterations() * fodder_size * kRounds);
 }
 
 static void ArrayDataConstructDestruct(
@@ -371,7 +370,7 @@ BENCHMARK(ReferenceBuildVectorNoNulls);
 
 BENCHMARK(BuildBooleanArrayNoNulls);
 
-BENCHMARK(BuildPrimitiveArrayNoNulls);
+BENCHMARK(BuildIntArrayNoNulls);
 BENCHMARK(BuildAdaptiveIntNoNulls);
 BENCHMARK(BuildAdaptiveIntNoNullsScalarAppend);
 

cpp/src/arrow/json/parser-benchmark.cc

@@ -150,7 +150,7 @@ static void ReadJSONBlockWithSchemaSingleThread(
   BenchmarkReadJSONBlockWithSchema(state, false);
 }
 
-static void ReferenceReadJSONBlockWithSchemaMultiThread(
+static void ReadJSONBlockWithSchemaMultiThread(
     benchmark::State& state) {  // NOLINT non-const reference
   BenchmarkReadJSONBlockWithSchema(state, true);
 }
@@ -160,7 +160,7 @@ BENCHMARK(ChunkJSONLineDelimited);
 BENCHMARK(ParseJSONBlockWithSchema);
 
 BENCHMARK(ReadJSONBlockWithSchemaSingleThread);
-BENCHMARK(ReferenceReadJSONBlockWithSchemaMultiThread)->UseRealTime();
+BENCHMARK(ReadJSONBlockWithSchemaMultiThread)->UseRealTime();
 
 }  // namespace json
 }  // namespace arrow

cpp/src/arrow/util/bit-util-benchmark.cc

@@ -30,6 +30,8 @@ namespace arrow {
 
 namespace BitUtil {
 
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
 // A naive bitmap reader implementation, meant as a baseline against
 // internal::BitmapReader
 
@@ -82,6 +84,8 @@ class NaiveBitmapWriter {
   int64_t position_;
 };
 
+#endif
+
 static std::shared_ptr<Buffer> CreateRandomBuffer(int64_t nbytes) {
   std::shared_ptr<Buffer> buffer;
   ABORT_NOT_OK(AllocateBuffer(nbytes, &buffer));
@@ -174,18 +178,10 @@ static void BenchmarkGenerateBits(benchmark::State& state, int64_t nbytes) {
   state.SetBytesProcessed(state.iterations() * nbytes);
 }
 
-static void ReferenceNaiveBitmapReader(benchmark::State& state) {
-  BenchmarkBitmapReader<NaiveBitmapReader>(state, state.range(0));
-}
-
 static void BitmapReader(benchmark::State& state) {
   BenchmarkBitmapReader<internal::BitmapReader>(state, state.range(0));
 }
 
-static void ReferenceNaiveBitmapWriter(benchmark::State& state) {
-  BenchmarkBitmapWriter<NaiveBitmapWriter>(state, state.range(0));
-}
-
 static void BitmapWriter(benchmark::State& state) {
   BenchmarkBitmapWriter<internal::BitmapWriter>(state, state.range(0));
 }
@@ -249,12 +245,25 @@ static void CopyBitmapWithOffset(benchmark::State& state) {  // NOLINT non-const
   CopyBitmap<4>(state);
 }
 
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
+static void ReferenceNaiveBitmapReader(benchmark::State& state) {
+  BenchmarkBitmapReader<NaiveBitmapReader>(state, state.range(0));
+}
+
+static void ReferenceNaiveBitmapWriter(benchmark::State& state) {
+  BenchmarkBitmapWriter<NaiveBitmapWriter>(state, state.range(0));
+}
+
+BENCHMARK(ReferenceNaiveBitmapWriter)->Arg(kBufferSize);
+BENCHMARK(ReferenceNaiveBitmapReader)->Arg(kBufferSize);
+
+#endif
+
 BENCHMARK(CopyBitmapWithoutOffset)->Arg(kBufferSize);
 BENCHMARK(CopyBitmapWithOffset)->Arg(kBufferSize);
 
-BENCHMARK(ReferenceNaiveBitmapReader)->Arg(kBufferSize);
 BENCHMARK(BitmapReader)->Arg(kBufferSize);
-BENCHMARK(ReferenceNaiveBitmapWriter)->Arg(kBufferSize);
 BENCHMARK(BitmapWriter)->Arg(kBufferSize);
 
 BENCHMARK(FirstTimeBitmapWriter)->Arg(kBufferSize);

cpp/src/arrow/util/compression-benchmark.cc

@@ -29,6 +29,8 @@
 namespace arrow {
 namespace util {
 
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
 std::vector<uint8_t> MakeCompressibleData(int data_size) {
   // XXX This isn't a real-world corpus so doesn't really represent the
   // comparative qualities of the algorithms
@@ -126,8 +128,9 @@ static void StreamingCompression(Compression::type compression,
 }
 
 template <Compression::type COMPRESSION>
-static void StreamingCompression(benchmark::State& state) {  // NOLINT non-const reference
-  auto data = MakeCompressibleData(8 * 1024 * 1024);         // 8 MB
+static void ReferenceStreamingCompression(
+    benchmark::State& state) {                        // NOLINT non-const reference
+  auto data = MakeCompressibleData(8 * 1024 * 1024);  // 8 MB
 
   StreamingCompression(COMPRESSION, data, state);
 }
@@ -172,22 +175,24 @@ static void StreamingDecompression(
 }
 
 template <Compression::type COMPRESSION>
-static void StreamingDecompression(
+static void ReferenceStreamingDecompression(
     benchmark::State& state) {                        // NOLINT non-const reference
   auto data = MakeCompressibleData(8 * 1024 * 1024);  // 8 MB
 
   StreamingDecompression(COMPRESSION, data, state);
 }
 
-BENCHMARK_TEMPLATE(StreamingCompression, Compression::GZIP);
-BENCHMARK_TEMPLATE(StreamingCompression, Compression::BROTLI);
-BENCHMARK_TEMPLATE(StreamingCompression, Compression::ZSTD);
-BENCHMARK_TEMPLATE(StreamingCompression, Compression::LZ4);
+BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::GZIP);
+BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::BROTLI);
+BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::ZSTD);
+BENCHMARK_TEMPLATE(ReferenceStreamingCompression, Compression::LZ4);
 
-BENCHMARK_TEMPLATE(StreamingDecompression, Compression::GZIP);
-BENCHMARK_TEMPLATE(StreamingDecompression, Compression::BROTLI);
-BENCHMARK_TEMPLATE(StreamingDecompression, Compression::ZSTD);
-BENCHMARK_TEMPLATE(StreamingDecompression, Compression::LZ4);
+BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::GZIP);
+BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::BROTLI);
+BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::ZSTD);
+BENCHMARK_TEMPLATE(ReferenceStreamingDecompression, Compression::LZ4);
+
+#endif
 
 }  // namespace util
 }  // namespace arrow

cpp/src/arrow/util/thread-pool-benchmark.cc

@@ -73,20 +73,6 @@ struct Task {
   Workload workload_;
 };
 
-// This benchmark simply provides a baseline indicating the raw cost of our workload
-// depending on the workload size.  Number of items / second in this (serial)
-// benchmark can be compared to the numbers obtained in ThreadPoolSpawn.
-static void WorkloadCost(benchmark::State& state) {
-  const auto workload_size = static_cast<int32_t>(state.range(0));
-
-  Workload workload(workload_size);
-  for (auto _ : state) {
-    workload();
-  }
-
-  state.SetItemsProcessed(state.iterations());
-}
-
 // Benchmark ThreadPool::Spawn
 static void ThreadPoolSpawn(benchmark::State& state) {
   const auto nthreads = static_cast<int>(state.range(0));
@@ -169,7 +155,6 @@ static void WorkloadCost_Customize(benchmark::internal::Benchmark* b) {
   }
   b->ArgNames({"task_cost"});
   b->UseRealTime();
-  b->Repetitions(1);
 }
 
 static void ThreadPoolSpawn_Customize(benchmark::internal::Benchmark* b) {
@@ -180,10 +165,28 @@ static void ThreadPoolSpawn_Customize(benchmark::internal::Benchmark* b) {
   }
   b->ArgNames({"threads", "task_cost"});
   b->UseRealTime();
-  b->Repetitions(1);
 }
 
-BENCHMARK(WorkloadCost)->Apply(WorkloadCost_Customize);
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
+// This benchmark simply provides a baseline indicating the raw cost of our workload
+// depending on the workload size.  Number of items / second in this (serial)
+// benchmark can be compared to the numbers obtained in ThreadPoolSpawn.
+static void WorkloadCost(benchmark::State& state) {
+  const auto workload_size = static_cast<int32_t>(state.range(0));
+
+  Workload workload(workload_size);
+  for (auto _ : state) {
+    workload();
+  }
+
+  state.SetItemsProcessed(state.iterations());
+}
+
+BENCHMARK(ReferenceWorkloadCost)->Apply(WorkloadCost_Customize);
+
+#endif
+
 BENCHMARK(SerialTaskGroup)->Apply(WorkloadCost_Customize);
 BENCHMARK(ThreadPoolSpawn)->Apply(ThreadPoolSpawn_Customize);
 BENCHMARK(ThreadedTaskGroup)->Apply(ThreadPoolSpawn_Customize);

cpp/src/arrow/util/trie-benchmark.cc

@@ -28,6 +28,64 @@
 namespace arrow {
 namespace internal {
 
+std::vector<std::string> AllNulls() {
+  return {"#N/A",    "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND",
+          "1.#QNAN", "N/A",      "NA",  "NULL",    "NaN",      "n/a",  "nan",  "null"};
+}
+
+Trie MakeNullsTrie() {
+  auto nulls = AllNulls();
+
+  TrieBuilder builder;
+  for (const auto& str : AllNulls()) {
+    ABORT_NOT_OK(builder.Append(str));
+  }
+  return builder.Finish();
+}
+
+std::vector<std::string> Expand(const std::vector<std::string>& base, size_t n) {
+  std::vector<std::string> result;
+  result.reserve(n);
+
+  while (true) {
+    for (const auto& v : base) {
+      result.push_back(v);
+      if (result.size() == n) {
+        return result;
+      }
+    }
+  }
+}
+
+static void BenchmarkTrieLookups(benchmark::State& state,  // NOLINT non-const reference
+                                 const std::vector<std::string>& strings) {
+  Trie trie = MakeNullsTrie();
+  int32_t total = 0;
+
+  auto lookups = Expand(strings, 100);
+
+  for (auto _ : state) {
+    for (const auto& s : lookups) {
+      total += trie.Find(s);
+    }
+  }
+  benchmark::DoNotOptimize(total);
+  state.SetItemsProcessed(state.iterations() * lookups.size());
+}
+
+static void TrieLookupFound(benchmark::State& state) {  // NOLINT non-const reference
+  BenchmarkTrieLookups(state, {"N/A", "null", "-1.#IND", "N/A"});
+}
+
+static void TrieLookupNotFound(benchmark::State& state) {  // NOLINT non-const reference
+  BenchmarkTrieLookups(state, {"None", "1.0", "", "abc"});
+}
+
+BENCHMARK(TrieLookupFound);
+BENCHMARK(TrieLookupNotFound);
+
+#ifdef ARROW_WITH_BENCHMARKS_REFERENCE
+
 static inline bool InlinedNullLookup(util::string_view s) {
   // An inlined version of trie lookup for a specific set of strings
   // (see AllNulls())
@@ -130,51 +188,6 @@ static inline bool InlinedNullLookup(util::string_view s) {
   }
 }
 
-std::vector<std::string> AllNulls() {
-  return {"#N/A",    "#N/A N/A", "#NA", "-1.#IND", "-1.#QNAN", "-NaN", "-nan", "1.#IND",
-          "1.#QNAN", "N/A",      "NA",  "NULL",    "NaN",      "n/a",  "nan",  "null"};
-}
-
-Trie MakeNullsTrie() {
-  auto nulls = AllNulls();
-
-  TrieBuilder builder;
-  for (const auto& str : AllNulls()) {
-    ABORT_NOT_OK(builder.Append(str));
-  }
-  return builder.Finish();
-}
-
-std::vector<std::string> Expand(const std::vector<std::string>& base, size_t n) {
-  std::vector<std::string> result;
-  result.reserve(n);
-
-  while (true) {
-    for (const auto& v : base) {
-      result.push_back(v);
-      if (result.size() == n) {
-        return result;
-      }
-    }
-  }
-}
-
-static void BenchmarkTrieLookups(benchmark::State& state,  // NOLINT non-const reference
-                                 const std::vector<std::string>& strings) {
-  Trie trie = MakeNullsTrie();
-  int32_t total = 0;
-
-  auto lookups = Expand(strings, 100);
-
-  for (auto _ : state) {
-    for (const auto& s : lookups) {
-      total += trie.Find(s);
-    }
-  }
-  benchmark::DoNotOptimize(total);
-  state.SetItemsProcessed(state.iterations() * lookups.size());
-}
-
 static void BenchmarkInlinedTrieLookups(
     benchmark::State& state,  // NOLINT non-const reference
     const std::vector<std::string>& strings) {
@@ -190,15 +203,6 @@ static void BenchmarkInlinedTrieLookups(
   benchmark::DoNotOptimize(total);
   state.SetItemsProcessed(state.iterations() * lookups.size());
 }
-
-static void TrieLookupFound(benchmark::State& state) {  // NOLINT non-const reference
-  BenchmarkTrieLookups(state, {"N/A", "null", "-1.#IND", "N/A"});
-}
-
-static void TrieLookupNotFound(benchmark::State& state) {  // NOLINT non-const reference
-  BenchmarkTrieLookups(state, {"None", "1.0", "", "abc"});
-}
-
 static void InlinedTrieLookupFound(
     benchmark::State& state) {  // NOLINT non-const reference
   BenchmarkInlinedTrieLookups(state, {"N/A", "null", "-1.#IND", "N/A"});
@@ -209,10 +213,10 @@ static void InlinedTrieLookupNotFound(
   BenchmarkInlinedTrieLookups(state, {"None", "1.0", "", "abc"});
 }
 
-BENCHMARK(TrieLookupFound);
-BENCHMARK(TrieLookupNotFound);
 BENCHMARK(InlinedTrieLookupFound);
 BENCHMARK(InlinedTrieLookupNotFound);
 
+#endif
+
 }  // namespace internal
 }  // namespace arrow

dev/archery/archery/benchmark/google.py

@@ -122,12 +122,7 @@ class GoogleBenchmarkObservation:
     @property
     def value(self):
         """ Return the benchmark value."""
-        if self.bytes_per_second:
-            return self.bytes_per_second
-        elif self.items_per_second:
-            return self.items_per_second
-        else:
-            return self.time
+        return self.bytes_per_second or self.items_per_second or self.time
 
     @property
     def unit(self):

dev/archery/archery/benchmark/runner.py

@@ -85,6 +85,12 @@ class StaticBenchmarkRunner(BenchmarkRunner):
         self._suites = suites
         super().__init__(**kwargs)
 
+    @property
+    def list_benchmarks(self):
+        for suite in self._suites:
+            for benchmark in suite.benchmarks:
+                yield f"{suite.name}.{benchmark.name}"
+
     @property
     def suites(self):
         suite_fn = regex_filter(self.suite_filter)
@@ -147,7 +153,7 @@ class CppBenchmarkRunner(BenchmarkRunner):
         return BenchmarkSuite(name, benchmarks)
 
     @property
-    def list(self):
+    def list_benchmarks(self):
         for suite_name, suite_bin in self.suites_binaries.items():
             suite_cmd = GoogleBenchmarkCommand(suite_bin)
             for benchmark_name in suite_cmd.list_benchmarks():

dev/archery/archery/cli.py

@@ -186,24 +186,24 @@ def benchmark(ctx):
 @click.option("--cmake-extras", type=str, multiple=True,
               help="Extra flags/options to pass to cmake invocation. "
               "Can be stacked")
-@click.argument("baseline", metavar="[<baseline>]", default="WORKSPACE",
+@click.argument("rev_or_path", metavar="[<rev_or_path>]", default="WORKSPACE",
                 required=False)
 @click.pass_context
-def benchmark_list(ctx, src, preserve, output, cmake_extras, baseline):
+def benchmark_list(ctx, src, preserve, output, cmake_extras, rev_or_path):
     """ List benchmark suite.
     """
     with tmpdir(preserve) as root:
-        logger.debug(f"Running benchmark {baseline}")
+        logger.debug(f"Running benchmark {rev_or_path}")
 
         conf = CppConfiguration(
             build_type="release", with_tests=True, with_benchmarks=True,
             with_python=False, cmake_extras=cmake_extras)
 
         runner_base = BenchmarkRunner.from_rev_or_path(
-            src, root, baseline, conf)
+            src, root, rev_or_path, conf)
 
-        for b in runner_base.list:
-            print(b, file=output)
+        for b in runner_base.list_benchmarks:
+            click.echo(b, file=output)
 
 
 @benchmark.command(name="run", short_help="Run benchmark suite")
@@ -224,11 +224,11 @@ def benchmark_list(ctx, src, preserve, output, cmake_extras, baseline):
 @click.option("--cmake-extras", type=str, multiple=True,
               help="Extra flags/options to pass to cmake invocation. "
               "Can be stacked")
-@click.argument("baseline", metavar="[<baseline>]", default="WORKSPACE",
+@click.argument("rev_or_path", metavar="[<rev_or_path>]", default="WORKSPACE",
                 required=False)
 @click.pass_context
 def benchmark_run(ctx, src, preserve, suite_filter, benchmark_filter,
-                  output, cmake_extras, baseline):
+                  output, cmake_extras, rev_or_path):
     """ Run benchmark suite.
 
     This command will run the benchmark suite for a single build. This is
@@ -237,7 +237,6 @@ def benchmark_run(ctx, src, preserve, suite_filter, benchmark_filter,
     The caller can optionally specify a target which is either a git revision
     (commit, tag, special values like HEAD) or a cmake build directory.
 
-
     When a commit is referenced, a local clone of the arrow sources (specified
     via --src) is performed and the proper branch is created. This is done in
     a temporary directory which can be left intact with the `---preserve` flag.
@@ -263,14 +262,14 @@ def benchmark_run(ctx, src, preserve, suite_filter, benchmark_filter,
     archery benchmark run --output=run.json
     """
     with tmpdir(preserve) as root:
-        logger.debug(f"Running benchmark {baseline}")
+        logger.debug(f"Running benchmark {rev_or_path}")
 
         conf = CppConfiguration(
             build_type="release", with_tests=True, with_benchmarks=True,
             with_python=False, cmake_extras=cmake_extras)
 
         runner_base = BenchmarkRunner.from_rev_or_path(
-            src, root, baseline, conf,
+            src, root, rev_or_path, conf,
             suite_filter=suite_filter, benchmark_filter=benchmark_filter)
 
         json.dump(runner_base, output, cls=JsonEncoder)