From 3af5f8a15314446d2b28a90e429c938dc7b8beeb Mon Sep 17 00:00:00 2001 From: 1ndahous3 <9205230+1ndahous3@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:25:57 +0300 Subject: [PATCH] Run benchmark workloads with consistent script namespaces (#9027) Assisted-by: Codex:gpt-6 --- benches/README.md | 11 ++++++---- benches/execution.rs | 14 +++++++++++-- benches/microbenchmarks.rs | 43 ++++++++++++++++++++------------------ 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/benches/README.md b/benches/README.md index d31e1a0f2..cdea0b3b4 100644 --- a/benches/README.md +++ b/benches/README.md @@ -42,9 +42,9 @@ Two things differ when the benchmarks run under CodSpeed: - The CPython comparison benchmarks are skipped. They are useful to compare RustPython against CPython locally, but CodSpeed tracks the evolution of RustPython itself, and running them would double the duration of an already slow instrumented run. -- The microbenchmarks using `ITERATIONS` run with a single value instead of five. The criterion - benchmark id does not include the iteration count, so all five sizes are reported under the same - name. +- The microbenchmarks using `ITERATIONS` run with a single value of 1,000 instead of five sizes. + CodSpeed retains the existing benchmark names; local runs include the iteration count in each + benchmark id. ### Adding a benchmark @@ -54,6 +54,9 @@ in two ways: 1. The time to parse the file to AST 2. The time it takes to execute the file +Execution includes compilation and runs each iteration with a fresh namespace whose `__name__` +is `"__main__"`, so script entry points run in both RustPython and CPython. + ### Adding a micro benchmark Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has @@ -73,7 +76,7 @@ len(a_list) ``` Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If -present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For +present then the benchmark runs at five sizes: 100, 300, 500, 700 and 900 iterations locally. For example: ```python diff --git a/benches/execution.rs b/benches/execution.rs index 56d4fc4fc..a09ab9c78 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -7,6 +7,7 @@ use criterion::{ Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, measurement::WallTime, }; +use pyo3::types::{PyCode, PyCodeInput, PyCodeMethods, PyDict, PyDictMethods}; use rustpython_compiler::Mode; use rustpython_vm::{Interpreter, PyResult, Settings}; use std::{collections::HashMap, path::Path}; @@ -25,9 +26,14 @@ fn bench_cpython_code(b: &mut Bencher<'_>, source: &str) { let c_str_source = c_str_source_head.as_c_str(); pyo3::Python::attach(|py| { b.iter(|| { - let module = pyo3::types::PyModule::from_code(py, c_str_source, c"", c"") + let code = PyCode::compile(py, c_str_source, c"", PyCodeInput::File) + .expect("Error compiling source"); + let globals = PyDict::new(py); + globals.set_item("__name__", "__main__").unwrap(); + let result = code + .run(Some(&globals), None) .expect("Error running source"); - black_box(module); + black_box(result); }) }) } @@ -46,6 +52,10 @@ fn bench_rustpython_code(b: &mut Bencher<'_>, name: &str, source: &str) { b.iter(|| { let code = vm.compile(source, Mode::Exec, name).unwrap(); let scope = vm.new_scope_with_builtins(); + scope + .globals + .set_item("__name__", vm.ctx.new_str("__main__").into(), vm) + .unwrap(); let res: PyResult = vm.run_code_obj(code, scope); vm.unwrap_pyresult(res); }) diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 4d48278e4..1c3bfea55 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -45,11 +45,8 @@ fn bench_cpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &MicroBen pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let exec = builtins.getattr("exec").expect("no exec in builtins"); - let bench_func = |(globals, locals): &mut ( - pyo3::Bound<'_, pyo3::types::PyDict>, - pyo3::Bound<'_, pyo3::types::PyDict>, - )| { - let res = exec.call((&code, &*globals, &*locals), None); + let bench_func = |globals: &mut pyo3::Bound<'_, pyo3::types::PyDict>| { + let res = exec.call((&code, &*globals), None); if let Err(e) = res { e.print(py); panic!("Error running microbenchmark") @@ -58,23 +55,24 @@ fn bench_cpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &MicroBen let bench_setup = |iterations| { let globals = pyo3::types::PyDict::new(py); - let locals = pyo3::types::PyDict::new(py); + globals.set_item("__name__", "__main__").unwrap(); if let Some(idx) = iterations { globals.set_item("ITERATIONS", idx).unwrap(); } - let res = exec.call((&setup_code, &globals, &locals), None); + let res = exec.call((&setup_code, &globals), None); if let Err(e) = res { e.print(py); panic!("Error running microbenchmark setup code") } - (globals, locals) + globals }; if bench.iterate { for idx in iteration_counts() { group.throughput(Throughput::Elements(idx as u64)); - group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| { + let id = BenchmarkId::new("cpython", format!("{}[{idx}]", bench.name)); + group.bench_with_input(id, &idx, |b, idx| { b.iter_batched_ref( || bench_setup(Some(*idx)), bench_func, @@ -128,6 +126,10 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &Micro let bench_setup = |iterations| { let scope = vm.new_scope_with_builtins(); + scope + .globals + .set_item("__name__", vm.ctx.new_str("__main__").into(), vm) + .unwrap(); if let Some(idx) = iterations { scope .locals @@ -145,17 +147,18 @@ fn bench_rustpython_code(group: &mut BenchmarkGroup<'_, WallTime>, bench: &Micro if bench.iterate { for idx in iteration_counts() { group.throughput(Throughput::Elements(idx as u64)); - group.bench_with_input( - BenchmarkId::new("rustpython", &bench.name), - &idx, - |b, idx| { - b.iter_batched( - || bench_setup(Some(*idx)), - bench_func, - BatchSize::LargeInput, - ); - }, - ); + let name = if is_codspeed() { + bench.name.clone() + } else { + format!("{}[{idx}]", bench.name) + }; + group.bench_with_input(BenchmarkId::new("rustpython", name), &idx, |b, idx| { + b.iter_batched( + || bench_setup(Some(*idx)), + bench_func, + BatchSize::LargeInput, + ); + }); } } else { group.bench_function(BenchmarkId::new("rustpython", &bench.name), move |b| {