Add CodSpeed continuous benchmarking (#8613)

Alias criterion to codspeed-criterion-compat so the existing benchmark suites can be measured by CodSpeed with CPU simulation on every pull request, and add the workflow running them.

Co-authored-by: CodSpeed Bot <no-reply@codspeed.io>

benches/README.md

@@ -21,6 +21,31 @@ specific installed Python version by running:
 PYTHON_SYS_EXECUTABLE=python3.13 cargo bench
 ```
 
+## Continuous benchmarking with CodSpeed
+
+The benchmarks are also run on every pull request by
+[CodSpeed](https://app.codspeed.io/RustPython/RustPython), which measures them with CPU simulation
+instead of wall time. `criterion` is aliased to `codspeed-criterion-compat`, so nothing changes for
+`cargo bench`: the compatibility layer only takes over when the CodSpeed runner drives the
+benchmarks.
+
+To reproduce a CodSpeed run locally, install [`cargo-codspeed`](https://crates.io/crates/cargo-codspeed)
+and the [CodSpeed CLI](https://codspeed.io/docs/cli), then run:
+
+```shell
+cargo codspeed build -p rustpython -p rustpython-sre_engine
+codspeed run --mode simulation -- cargo codspeed run -p rustpython -p rustpython-sre_engine
+```
+
+Two things differ when the benchmarks run under CodSpeed:
+
+- The CPython comparison benchmarks are skipped. They are useful to compare RustPython against
+  CPython locally, but CodSpeed tracks the evolution of RustPython itself, and running them would
+  double the duration of an already slow instrumented run.
+- The microbenchmarks using `ITERATIONS` run with a single value instead of five. The criterion
+  benchmark id does not include the iteration count, so all five sizes are reported under the same
+  name.
+
 ### Adding a benchmark
 
 Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested

benches/execution.rs

@@ -6,6 +6,15 @@ use rustpython_compiler::Mode;
 use rustpython_vm::{Interpreter, PyResult, Settings};
 use std::{collections::HashMap, hint::black_box, path::Path};
 
+/// `true` when the benchmarks are executed by the CodSpeed runner.
+///
+/// CodSpeed tracks the performance of RustPython itself, so the CPython
+/// reference benchmarks are skipped there: they double the (already slow)
+/// instrumented run without ever reporting a change of RustPython.
+fn is_codspeed() -> bool {
+    std::env::var_os("CODSPEED_ENV").is_some()
+}
+
 fn bench_cpython_code(b: &mut Bencher, source: &str) {
     let c_str_source_head = std::ffi::CString::new(source).unwrap();
     let c_str_source = c_str_source_head.as_c_str();
@@ -39,9 +48,11 @@ fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) {
 }
 
 pub fn benchmark_file_execution(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) {
-    group.bench_function(BenchmarkId::new(name, "cpython"), |b| {
-        bench_cpython_code(b, contents)
-    });
+    if !is_codspeed() {
+        group.bench_function(BenchmarkId::new(name, "cpython"), |b| {
+            bench_cpython_code(b, contents)
+        });
+    }
     group.bench_function(BenchmarkId::new(name, "rustpython"), |b| {
         bench_rustpython_code(b, name, contents)
     });
@@ -52,20 +63,22 @@ pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str,
     group.bench_function(BenchmarkId::new("rustpython", name), |b| {
         b.iter(|| ruff_python_parser::parse_module(contents).unwrap())
     });
-    group.bench_function(BenchmarkId::new("cpython", name), |b| {
-        use pyo3::types::PyAnyMethods;
-        pyo3::Python::attach(|py| {
-            let builtins =
-                pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins");
-            let compile = builtins.getattr("compile").expect("no compile in builtins");
-            b.iter(|| {
-                let x = compile
-                    .call1((contents, name, "exec"))
-                    .expect("Failed to parse code");
-                black_box(x);
+    if !is_codspeed() {
+        group.bench_function(BenchmarkId::new("cpython", name), |b| {
+            use pyo3::types::PyAnyMethods;
+            pyo3::Python::attach(|py| {
+                let builtins = pyo3::types::PyModule::import(py, "builtins")
+                    .expect("Failed to import builtins");
+                let compile = builtins.getattr("compile").expect("no compile in builtins");
+                b.iter(|| {
+                    let x = compile
+                        .call1((contents, name, "exec"))
+                        .expect("Failed to parse code");
+                    black_box(x);
+                })
             })
-        })
-    });
+        });
+    }
 }
 
 pub fn benchmark_pystone(group: &mut BenchmarkGroup<WallTime>, contents: String) {
@@ -75,9 +88,11 @@ pub fn benchmark_pystone(group: &mut BenchmarkGroup<WallTime>, contents: String)
         let code_str = code_with_loops.as_str();
 
         group.throughput(Throughput::Elements(idx as u64));
-        group.bench_function(BenchmarkId::new("cpython", idx), |b| {
-            bench_cpython_code(b, code_str)
-        });
+        if !is_codspeed() {
+            group.bench_function(BenchmarkId::new("cpython", idx), |b| {
+                bench_cpython_code(b, code_str)
+            });
+        }
         group.bench_function(BenchmarkId::new("rustpython", idx), |b| {
             bench_rustpython_code(b, "pystone", code_str)
         });

benches/microbenchmarks.rs

@@ -75,7 +75,7 @@ fn bench_cpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchma
         };
 
         if bench.iterate {
-            for idx in (100..=1_000).step_by(200) {
+            for idx in iteration_counts() {
                 group.throughput(Throughput::Elements(idx as u64));
                 group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| {
                     b.iter_batched_ref(
@@ -146,7 +146,7 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc
         };
 
         if bench.iterate {
-            for idx in (100..=1_000).step_by(200) {
+            for idx in iteration_counts() {
                 group.throughput(Throughput::Elements(idx as u64));
                 group.bench_with_input(
                     BenchmarkId::new("rustpython", &bench.name),
@@ -168,10 +168,34 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc
     })
 }
 
+/// `true` when the benchmarks are executed by the CodSpeed runner.
+///
+/// CodSpeed tracks the performance of RustPython itself, so the CPython
+/// reference benchmarks are skipped there: they double the (already slow)
+/// instrumented run without ever reporting a change of RustPython.
+fn is_codspeed() -> bool {
+    std::env::var_os("CODSPEED_ENV").is_some()
+}
+
+/// The `ITERATIONS` values the benchmarks referencing them are run with.
+///
+/// A single size is used under CodSpeed: the criterion benchmark id does not
+/// include the iteration count, so every size would be reported under the same
+/// name.
+fn iteration_counts() -> Vec<i32> {
+    if is_codspeed() {
+        vec![1_000]
+    } else {
+        (100..=1_000).step_by(200).collect()
+    }
+}
+
 pub fn run_micro_benchmark(c: &mut Criterion, benchmark: MicroBenchmark) {
     let mut group = c.benchmark_group("microbenchmarks");
 
-    bench_cpython_code(&mut group, &benchmark);
+    if !is_codspeed() {
+        bench_cpython_code(&mut group, &benchmark);
+    }
     bench_rustpython_code(&mut group, &benchmark);
 
     group.finish();