diff --git a/.github/workflows/codspeed.yaml b/.github/workflows/codspeed.yaml new file mode 100644 index 000000000..00035dca0 --- /dev/null +++ b/.github/workflows/codspeed.yaml @@ -0,0 +1,77 @@ +on: + push: + branches: [main] + pull_request: + types: [unlabeled, opened, synchronize, reopened] + merge_group: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +name: CodSpeed + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + CARGO_INCREMENTAL: 0 + CARGO_TERM_COLOR: always + CI: true + +jobs: + benchmarks: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} + name: Run benchmarks + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + id-token: write # for OpenID Connect authentication with CodSpeed + env: + CACHE_PATHS: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: dtolnay/rust-toolchain@stable + + # The benchmarks are linked against libpython through the pyo3 dev-dependency. + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + + - name: Restore cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.CACHE_PATHS }} + key: codspeed-${{ hashFiles('**/Cargo.lock') }} + restore-keys: codspeed- + continue-on-error: true + + - name: Install cargo-codspeed + run: cargo install cargo-codspeed --locked --version ^5 + + - name: Build the benchmark targets + run: cargo codspeed build --locked --measurement-mode simulation -p rustpython -p rustpython-sre_engine + + - name: Save cache + if: ${{ github.ref == 'refs/heads/main' }} # only save on main + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.CACHE_PATHS }} + key: codspeed-${{ hashFiles('**/Cargo.lock') }} + continue-on-error: true + + - name: Run the benchmarks + uses: CodSpeedHQ/action@373d6868929f444bc08d901fd0eb0ad52a8875ea # v5.2.1 + with: + mode: simulation + run: cargo codspeed run -p rustpython -p rustpython-sre_engine diff --git a/Cargo.lock b/Cargo.lock index cc1ad30a8..21f7387de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,15 +58,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloca" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" -dependencies = [ - "cc", -] - [[package]] name = "allocator-api2" version = "0.2.21" @@ -598,6 +589,64 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "codspeed" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7083f253260bcb4aaa3b4aa4c52973703dabc1a85c2f193997e2689aafa8a919" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.4.3", + "glob", + "libc", + "nix 0.31.3", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f24251445188c69d50f10179d795424bd71b533b7e3f84fc03f26893b01af0" +dependencies = [ + "clap", + "codspeed", + "codspeed-criterion-compat-walltime", + "colored", + "regex", +] + +[[package]] +name = "codspeed-criterion-compat-walltime" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c38205d56e2cb4fe04b708de7f9653a3f1b89edbe3a20b28f21e9e525e9e061" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "codspeed", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + [[package]] name = "collection_literals" version = "1.0.3" @@ -610,6 +659,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "combine" version = "4.6.7" @@ -896,39 +954,14 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "criterion" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" -dependencies = [ - "alloca", - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "itertools 0.13.0", - "num-traits", - "oorandom", - "page_size", - "plotters", - "rayon", - "regex", - "serde", - "serde_json", - "tinytemplate", - "walkdir", -] - [[package]] name = "criterion-plot" -version = "0.8.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.13.0", + "itertools 0.10.5", ] [[package]] @@ -1727,12 +1760,32 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -2464,16 +2517,6 @@ dependencies = [ "indexmap", ] -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking_lot" version = "0.12.5" @@ -3253,7 +3296,7 @@ dependencies = [ name = "rustpython" version = "0.5.0" dependencies = [ - "criterion", + "codspeed-criterion-compat", "dirs", "env_logger", "flame", @@ -3543,7 +3586,7 @@ name = "rustpython-sre_engine" version = "0.5.0" dependencies = [ "bitflags 2.13.1", - "criterion", + "codspeed-criterion-compat", "num_enum", "optional", "rustpython-unicode", @@ -4069,6 +4112,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strum" version = "0.28.0" diff --git a/Cargo.toml b/Cargo.toml index bf44e55d9..f2d1579f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,7 +206,7 @@ cranelift = "0.132.0" cranelift-jit = "0.132.0" cranelift-module = "0.132.0" crc32fast = "1.3.2" -criterion = { version = "0.8", features = ["html_reports"] } +criterion = { package = "codspeed-criterion-compat", version = "5.0.1", features = ["html_reports"] } crossbeam-utils = "0.8.21" csv-core = "0.1.11" digest = "0.11" diff --git a/README.md b/README.md index 9da54c4b4..1c8c972ed 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ A Python-3 (CPython >= 3.14.0) Interpreter written in Rust :snake: :scream: [![Build Status](https://github.com/RustPython/RustPython/workflows/CI/badge.svg)](https://github.com/RustPython/RustPython/actions?query=workflow%3ACI) [![codecov](https://codecov.io/gh/RustPython/RustPython/branch/main/graph/badge.svg)](https://codecov.io/gh/RustPython/RustPython) +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://app.codspeed.io/RustPython/RustPython?utm_source=badge) [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](https://opensource.org/licenses/MIT) [![Contributors](https://img.shields.io/github/contributors/RustPython/RustPython.svg)](https://github.com/RustPython/RustPython/graphs/contributors) [![Discord Shield](https://discordapp.com/api/guilds/1043121930691149845/widget.png?style=shield)][discord] diff --git a/benches/README.md b/benches/README.md index 4823751fd..990ea2139 100644 --- a/benches/README.md +++ b/benches/README.md @@ -21,6 +21,31 @@ specific installed Python version by running: PYTHON_SYS_EXECUTABLE=python3.13 cargo bench ``` +## Continuous benchmarking with CodSpeed + +The benchmarks are also run on every pull request by +[CodSpeed](https://app.codspeed.io/RustPython/RustPython), which measures them with CPU simulation +instead of wall time. `criterion` is aliased to `codspeed-criterion-compat`, so nothing changes for +`cargo bench`: the compatibility layer only takes over when the CodSpeed runner drives the +benchmarks. + +To reproduce a CodSpeed run locally, install [`cargo-codspeed`](https://crates.io/crates/cargo-codspeed) +and the [CodSpeed CLI](https://codspeed.io/docs/cli), then run: + +```shell +cargo codspeed build -p rustpython -p rustpython-sre_engine +codspeed run --mode simulation -- cargo codspeed run -p rustpython -p rustpython-sre_engine +``` + +Two things differ when the benchmarks run under CodSpeed: + +- The CPython comparison benchmarks are skipped. They are useful to compare RustPython against + CPython locally, but CodSpeed tracks the evolution of RustPython itself, and running them would + double the duration of an already slow instrumented run. +- The microbenchmarks using `ITERATIONS` run with a single value instead of five. The criterion + benchmark id does not include the iteration count, so all five sizes are reported under the same + name. + ### Adding a benchmark Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested diff --git a/benches/execution.rs b/benches/execution.rs index 0eef88c8b..0152de240 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -6,6 +6,15 @@ use rustpython_compiler::Mode; use rustpython_vm::{Interpreter, PyResult, Settings}; use std::{collections::HashMap, hint::black_box, path::Path}; +/// `true` when the benchmarks are executed by the CodSpeed runner. +/// +/// CodSpeed tracks the performance of RustPython itself, so the CPython +/// reference benchmarks are skipped there: they double the (already slow) +/// instrumented run without ever reporting a change of RustPython. +fn is_codspeed() -> bool { + std::env::var_os("CODSPEED_ENV").is_some() +} + fn bench_cpython_code(b: &mut Bencher, source: &str) { let c_str_source_head = std::ffi::CString::new(source).unwrap(); let c_str_source = c_str_source_head.as_c_str(); @@ -39,9 +48,11 @@ fn bench_rustpython_code(b: &mut Bencher, name: &str, source: &str) { } pub fn benchmark_file_execution(group: &mut BenchmarkGroup, name: &str, contents: &str) { - group.bench_function(BenchmarkId::new(name, "cpython"), |b| { - bench_cpython_code(b, contents) - }); + if !is_codspeed() { + group.bench_function(BenchmarkId::new(name, "cpython"), |b| { + bench_cpython_code(b, contents) + }); + } group.bench_function(BenchmarkId::new(name, "rustpython"), |b| { bench_rustpython_code(b, name, contents) }); @@ -52,20 +63,22 @@ pub fn benchmark_file_parsing(group: &mut BenchmarkGroup, name: &str, group.bench_function(BenchmarkId::new("rustpython", name), |b| { b.iter(|| ruff_python_parser::parse_module(contents).unwrap()) }); - group.bench_function(BenchmarkId::new("cpython", name), |b| { - use pyo3::types::PyAnyMethods; - pyo3::Python::attach(|py| { - let builtins = - pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); - let compile = builtins.getattr("compile").expect("no compile in builtins"); - b.iter(|| { - let x = compile - .call1((contents, name, "exec")) - .expect("Failed to parse code"); - black_box(x); + if !is_codspeed() { + group.bench_function(BenchmarkId::new("cpython", name), |b| { + use pyo3::types::PyAnyMethods; + pyo3::Python::attach(|py| { + let builtins = pyo3::types::PyModule::import(py, "builtins") + .expect("Failed to import builtins"); + let compile = builtins.getattr("compile").expect("no compile in builtins"); + b.iter(|| { + let x = compile + .call1((contents, name, "exec")) + .expect("Failed to parse code"); + black_box(x); + }) }) - }) - }); + }); + } } pub fn benchmark_pystone(group: &mut BenchmarkGroup, contents: String) { @@ -75,9 +88,11 @@ pub fn benchmark_pystone(group: &mut BenchmarkGroup, contents: String) let code_str = code_with_loops.as_str(); group.throughput(Throughput::Elements(idx as u64)); - group.bench_function(BenchmarkId::new("cpython", idx), |b| { - bench_cpython_code(b, code_str) - }); + if !is_codspeed() { + group.bench_function(BenchmarkId::new("cpython", idx), |b| { + bench_cpython_code(b, code_str) + }); + } group.bench_function(BenchmarkId::new("rustpython", idx), |b| { bench_rustpython_code(b, "pystone", code_str) }); diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 34e0b6846..9f7f70c02 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -75,7 +75,7 @@ fn bench_cpython_code(group: &mut BenchmarkGroup, bench: &MicroBenchma }; if bench.iterate { - for idx in (100..=1_000).step_by(200) { + for idx in iteration_counts() { group.throughput(Throughput::Elements(idx as u64)); group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| { b.iter_batched_ref( @@ -146,7 +146,7 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup, bench: &MicroBenc }; if bench.iterate { - for idx in (100..=1_000).step_by(200) { + for idx in iteration_counts() { group.throughput(Throughput::Elements(idx as u64)); group.bench_with_input( BenchmarkId::new("rustpython", &bench.name), @@ -168,10 +168,34 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup, bench: &MicroBenc }) } +/// `true` when the benchmarks are executed by the CodSpeed runner. +/// +/// CodSpeed tracks the performance of RustPython itself, so the CPython +/// reference benchmarks are skipped there: they double the (already slow) +/// instrumented run without ever reporting a change of RustPython. +fn is_codspeed() -> bool { + std::env::var_os("CODSPEED_ENV").is_some() +} + +/// The `ITERATIONS` values the benchmarks referencing them are run with. +/// +/// A single size is used under CodSpeed: the criterion benchmark id does not +/// include the iteration count, so every size would be reported under the same +/// name. +fn iteration_counts() -> Vec { + if is_codspeed() { + vec![1_000] + } else { + (100..=1_000).step_by(200).collect() + } +} + pub fn run_micro_benchmark(c: &mut Criterion, benchmark: MicroBenchmark) { let mut group = c.benchmark_group("microbenchmarks"); - bench_cpython_code(&mut group, &benchmark); + if !is_codspeed() { + bench_cpython_code(&mut group, &benchmark); + } bench_rustpython_code(&mut group, &benchmark); group.finish();