Add CodSpeed continuous benchmarking (#8613)

Alias criterion to codspeed-criterion-compat so the existing benchmark suites can be measured by CodSpeed with CPU simulation on every pull request, and add the workflow running them. Co-authored-by: CodSpeed Bot <no-reply@codspeed.io>
2026-10-28 18:59:04 +08:00 · 2026-08-30 15:18:33 +09:00
1 parent 7c6c78f955
commit 062bd72c60
@@ -0,0 +1,77 @@
+on:
+  push:
+    branches: [main]
+  pull_request:
+    types: [unlabeled, opened, synchronize, reopened]
+  merge_group:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+name: CodSpeed
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: true
+
+env:
+  CARGO_INCREMENTAL: 0
+  CARGO_TERM_COLOR: always
+  CI: true
+
+jobs:
+  benchmarks:
+    if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }}
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    permissions:
+      contents: read
+      id-token: write # for OpenID Connect authentication with CodSpeed
+    env:
+      CACHE_PATHS: |
+        ~/.cargo/bin/
+        ~/.cargo/registry/index/
+        ~/.cargo/registry/cache/
+        ~/.cargo/git/db/
+        target/
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      # The benchmarks are linked against libpython through the pyo3 dev-dependency.
+      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+
+      - name: Restore cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.CACHE_PATHS }}
+          key: codspeed-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: codspeed-
+        continue-on-error: true
+
+      - name: Install cargo-codspeed
+        run: cargo install cargo-codspeed --locked --version ^5
+
+      - name: Build the benchmark targets
+        run: cargo codspeed build --locked --measurement-mode simulation -p rustpython -p rustpython-sre_engine
+
+      - name: Save cache
+        if: ${{ github.ref == 'refs/heads/main' }} # only save on main
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ${{ env.CACHE_PATHS }}
+          key: codspeed-${{ hashFiles('**/Cargo.lock') }}
+        continue-on-error: true
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1
+        with:
+          mode: simulation
+          run: cargo codspeed run -p rustpython -p rustpython-sre_engine
@@ -58,15 +58,6 @@ dependencies = [
 "memchr",
 ]

-[[package]]
-name = "alloca"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
-dependencies = [
- "cc",
-]
-
 [[package]]
 name = "allocator-api2"
 version = "0.2.21"
@@ -598,6 +589,64 @@ version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"

+[[package]]
+name = "codspeed"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7083f253260bcb4aaa3b4aa4c52973703dabc1a85c2f193997e2689aafa8a919"
+dependencies = [
+ "anyhow",
+ "cc",
+ "colored",
+ "getrandom 0.4.3",
+ "glob",
+ "libc",
+ "nix 0.31.3",
+ "serde",
+ "serde_json",
+ "statrs",
+]
+
+[[package]]
+name = "codspeed-criterion-compat"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9f24251445188c69d50f10179d795424bd71b533b7e3f84fc03f26893b01af0"
+dependencies = [
+ "clap",
+ "codspeed",
+ "codspeed-criterion-compat-walltime",
+ "colored",
+ "regex",
+]
+
+[[package]]
+name = "codspeed-criterion-compat-walltime"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c38205d56e2cb4fe04b708de7f9653a3f1b89edbe3a20b28f21e9e525e9e061"
+dependencies = [
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "codspeed",
+ "criterion-plot",
+ "is-terminal",
+ "itertools 0.10.5",
+ "num-traits",
+ "once_cell",
+ "oorandom",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
 [[package]]
 name = "collection_literals"
 version = "1.0.3"
@@ -610,6 +659,15 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"

+[[package]]
+name = "colored"
+version = "3.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -896,39 +954,14 @@ dependencies = [
 "cfg-if",
 ]

-[[package]]
-name = "criterion"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
-dependencies = [
- "alloca",
- "anes",
- "cast",
- "ciborium",
- "clap",
- "criterion-plot",
- "itertools 0.13.0",
- "num-traits",
- "oorandom",
- "page_size",
- "plotters",
- "rayon",
- "regex",
- "serde",
- "serde_json",
- "tinytemplate",
- "walkdir",
-]
-
 [[package]]
 name = "criterion-plot"
-version = "0.8.2"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
+checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1"
 dependencies = [
 "cast",
- "itertools 0.13.0",
+ "itertools 0.10.5",
 ]

 [[package]]
@@ -1727,12 +1760,32 @@ dependencies = [
 "syn 2.0.119",
 ]

+[[package]]
+name = "is-terminal"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "windows-sys 0.61.2",
+]
+
 [[package]]
 name = "is_terminal_polyfill"
 version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"

+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -2464,16 +2517,6 @@ dependencies = [
 "indexmap",
 ]

-[[package]]
-name = "page_size"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
-dependencies = [
- "libc",
- "winapi",
-]
-
 [[package]]
 name = "parking_lot"
 version = "0.12.5"
@@ -3253,7 +3296,7 @@ dependencies = [
 name = "rustpython"
 version = "0.5.0"
 dependencies = [
- "criterion",
+ "codspeed-criterion-compat",
 "dirs",
 "env_logger",
 "flame",
@@ -3543,7 +3586,7 @@ name = "rustpython-sre_engine"
 version = "0.5.0"
 dependencies = [
 "bitflags 2.13.1",
- "criterion",
+ "codspeed-criterion-compat",
 "num_enum",
 "optional",
 "rustpython-unicode",
@@ -4069,6 +4112,16 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"

+[[package]]
+name = "statrs"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e"
+dependencies = [
+ "approx",
+ "num-traits",
+]
+
 [[package]]
 name = "strum"
 version = "0.28.0"
@@ -206,7 +206,7 @@ cranelift = "0.132.0"
 cranelift-jit = "0.132.0"
 cranelift-module = "0.132.0"
 crc32fast = "1.3.2"
-criterion = { version = "0.8", features = ["html_reports"] }
+criterion = { package = "codspeed-criterion-compat", version = "5.0.1", features = ["html_reports"] }
 crossbeam-utils = "0.8.21"
 csv-core = "0.1.11"
 digest = "0.11"
@@ -7,6 +7,7 @@ A Python-3 (CPython >= 3.14.0) Interpreter written in Rust :snake: :scream:

 [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI)
 [![codecov](https://codecov.io/gh/RustPython/RustPython/branch/main/graph/badge.svg)](https://codecov.io/gh/RustPython/RustPython)
+[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/RustPython/RustPython?utm_source=badge)
 [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT)
 [![Contributors](https://img.shields.io/github/contributors/RustPython/RustPython.svg)](https://github.com/RustPython/RustPython/graphs/contributors)
 [![Discord Shield](https://discordapp.com/api/guilds/1043121930691149845/widget.png?style=shield)][discord]
@@ -21,6 +21,31 @@ specific installed Python version by running:
 PYTHON_SYS_EXECUTABLE=python3.13 cargo bench
 ```

+## Continuous benchmarking with CodSpeed
+
+The benchmarks are also run on every pull request by
+[CodSpeed](https://app.codspeed.io/RustPython/RustPython), which measures them with CPU simulation
+instead of wall time. `criterion` is aliased to `codspeed-criterion-compat`, so nothing changes for
+`cargo bench`: the compatibility layer only takes over when the CodSpeed runner drives the
+benchmarks.
+
+To reproduce a CodSpeed run locally, install [`cargo-codspeed`](https://crates.io/crates/cargo-codspeed)
+and the [CodSpeed CLI](https://codspeed.io/docs/cli), then run:
+
+```shell
+cargo codspeed build -p rustpython -p rustpython-sre_engine
+codspeed run --mode simulation -- cargo codspeed run -p rustpython -p rustpython-sre_engine
+```
+
+Two things differ when the benchmarks run under CodSpeed:
+
+- The CPython comparison benchmarks are skipped. They are useful to compare RustPython against
+  CPython locally, but CodSpeed tracks the evolution of RustPython itself, and running them would
+  double the duration of an already slow instrumented run.
+- The microbenchmarks using `ITERATIONS` run with a single value instead of five. The criterion
+  benchmark id does not include the iteration count, so all five sizes are reported under the same
+  name.
+
 ### Adding a benchmark

 Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested
@@ -6,6 +6,15 @@ use rustpython_compiler::Mode;
 use rustpython_vm::{Interpreter, PyResult, Settings};
 use std::{collections::HashMap, hint::black_box, path::Path};

+/// `true` when the benchmarks are executed by the CodSpeed runner.
+///
+/// CodSpeed tracks the performance of RustPython itself, so the CPython
+/// reference benchmarks are skipped there: they double the (already slow)
+/// instrumented run without ever reporting a change of RustPython.
+fn is_codspeed() -> bool {
+    std::env::var_os("CODSPEED_ENV").is_some()
+}
+
 fn bench_cpython_code(b: &mut Bencher, source: &str) {
    let c_str_source_head = std::ffi::CString::new(source).unwrap();
    let c_str_source = c_str_source_head.as_c_str();
@@ -39,9 +48,11 @@ fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) {
 }

 pub fn benchmark_file_execution(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &str) {
-    group.bench_function(BenchmarkId::new(name, "cpython"), |b| {
-        bench_cpython_code(b, contents)
-    });
+    if !is_codspeed() {
+        group.bench_function(BenchmarkId::new(name, "cpython"), |b| {
+            bench_cpython_code(b, contents)
+        });
+    }
    group.bench_function(BenchmarkId::new(name, "rustpython"), |b| {
        bench_rustpython_code(b, name, contents)
    });
@@ -52,20 +63,22 @@ pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str,
    group.bench_function(BenchmarkId::new("rustpython", name), |b| {
        b.iter(|| ruff_python_parser::parse_module(contents).unwrap())
    });
-    group.bench_function(BenchmarkId::new("cpython", name), |b| {
-        use pyo3::types::PyAnyMethods;
-        pyo3::Python::attach(|py| {
-            let builtins =
-                pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins");
-            let compile = builtins.getattr("compile").expect("no compile in builtins");
-            b.iter(|| {
-                let x = compile
-                    .call1((contents, name, "exec"))
-                    .expect("Failed to parse code");
-                black_box(x);
+    if !is_codspeed() {
+        group.bench_function(BenchmarkId::new("cpython", name), |b| {
+            use pyo3::types::PyAnyMethods;
+            pyo3::Python::attach(|py| {
+                let builtins = pyo3::types::PyModule::import(py, "builtins")
+                    .expect("Failed to import builtins");
+                let compile = builtins.getattr("compile").expect("no compile in builtins");
+                b.iter(|| {
+                    let x = compile
+                        .call1((contents, name, "exec"))
+                        .expect("Failed to parse code");
+                    black_box(x);
+                })
            })
-        })
-    });
+        });
+    }
 }

 pub fn benchmark_pystone(group: &mut BenchmarkGroup<WallTime>, contents: String) {
@@ -75,9 +88,11 @@ pub fn benchmark_pystone(group: &mut BenchmarkGroup<WallTime>, contents: String)
        let code_str = code_with_loops.as_str();

        group.throughput(Throughput::Elements(idx as u64));
-        group.bench_function(BenchmarkId::new("cpython", idx), |b| {
-            bench_cpython_code(b, code_str)
-        });
+        if !is_codspeed() {
+            group.bench_function(BenchmarkId::new("cpython", idx), |b| {
+                bench_cpython_code(b, code_str)
+            });
+        }
        group.bench_function(BenchmarkId::new("rustpython", idx), |b| {
            bench_rustpython_code(b, "pystone", code_str)
        });
@@ -75,7 +75,7 @@ fn bench_cpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchma
        };

        if bench.iterate {
-            for idx in (100..=1_000).step_by(200) {
+            for idx in iteration_counts() {
                group.throughput(Throughput::Elements(idx as u64));
                group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| {
                    b.iter_batched_ref(
@@ -146,7 +146,7 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc
        };

        if bench.iterate {
-            for idx in (100..=1_000).step_by(200) {
+            for idx in iteration_counts() {
                group.throughput(Throughput::Elements(idx as u64));
                group.bench_with_input(
                    BenchmarkId::new("rustpython", &bench.name),
@@ -168,10 +168,34 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenc
    })
 }

+/// `true` when the benchmarks are executed by the CodSpeed runner.
+///
+/// CodSpeed tracks the performance of RustPython itself, so the CPython
+/// reference benchmarks are skipped there: they double the (already slow)
+/// instrumented run without ever reporting a change of RustPython.
+fn is_codspeed() -> bool {
+    std::env::var_os("CODSPEED_ENV").is_some()
+}
+
+/// The `ITERATIONS` values the benchmarks referencing them are run with.
+///
+/// A single size is used under CodSpeed: the criterion benchmark id does not
+/// include the iteration count, so every size would be reported under the same
+/// name.
+fn iteration_counts() -> Vec<i32> {
+    if is_codspeed() {
+        vec![1_000]
+    } else {
+        (100..=1_000).step_by(200).collect()
+    }
+}
+
 pub fn run_micro_benchmark(c: &mut Criterion, benchmark: MicroBenchmark) {
    let mut group = c.benchmark_group("microbenchmarks");

-    bench_cpython_code(&mut group, &benchmark);
+    if !is_codspeed() {
+        bench_cpython_code(&mut group, &benchmark);
+    }
    bench_rustpython_code(&mut group, &benchmark);

    group.finish();