mirror of
https://github.com/RustPython/RustPython.git
synced 2026-10-28 18:59:04 +08:00
Merge pull request #2367 from orf/criterion
Refactor benchmarking suite to use Criterion and add microbenchmarks
This commit is contained in:
30 files changed
+680
-2181
No files matched your search
@@ -160,7 +160,7 @@ jobs:
|
||||
- name: install flake8
|
||||
run: python -m pip install flake8
|
||||
- name: run lint
|
||||
run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib --select=E9,F63,F7,F82 --show-source --statistics
|
||||
run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source --statistics
|
||||
miri:
|
||||
name: Run tests under miri
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
@@ -16,3 +16,4 @@ flame.txt
|
||||
flamescope.json
|
||||
/wapm.lock
|
||||
/wapm_packages
|
||||
/.cargo/config
|
||||
Generated
+185
@@ -243,6 +243,15 @@ dependencies = [
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cast"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0"
|
||||
dependencies = [
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.66"
|
||||
@@ -305,6 +314,12 @@ dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const_fn"
|
||||
version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826"
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.1.5"
|
||||
@@ -454,6 +469,77 @@ dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"cast",
|
||||
"clap",
|
||||
"criterion-plot",
|
||||
"csv",
|
||||
"itertools",
|
||||
"lazy_static 1.4.0",
|
||||
"num-traits",
|
||||
"oorandom",
|
||||
"plotters",
|
||||
"rayon",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_cbor",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"tinytemplate",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "criterion-plot"
|
||||
version = "0.4.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d"
|
||||
dependencies = [
|
||||
"cast",
|
||||
"itertools",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-deque"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"crossbeam-epoch",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
|
||||
dependencies = [
|
||||
"cfg-if 1.0.0",
|
||||
"const_fn",
|
||||
"crossbeam-utils",
|
||||
"lazy_static 1.4.0",
|
||||
"memoffset",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.1"
|
||||
@@ -802,6 +888,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.9.1"
|
||||
@@ -1093,6 +1185,15 @@ version = "2.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.3"
|
||||
@@ -1233,6 +1334,12 @@ version = "1.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0"
|
||||
|
||||
[[package]]
|
||||
name = "oorandom"
|
||||
version = "11.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.2.3"
|
||||
@@ -1351,6 +1458,18 @@ version = "0.3.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
|
||||
|
||||
[[package]]
|
||||
name = "plotters"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d1685fbe7beba33de0330629da9d955ac75bd54f33d7b79f9a895590124f6bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pmutil"
|
||||
version = "0.5.3"
|
||||
@@ -1475,6 +1594,31 @@ dependencies = [
|
||||
"rustc_version",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"crossbeam-deque",
|
||||
"either",
|
||||
"rayon-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
"crossbeam-utils",
|
||||
"lazy_static 1.4.0",
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.1.57"
|
||||
@@ -1595,6 +1739,7 @@ dependencies = [
|
||||
"cfg-if 0.1.10",
|
||||
"clap",
|
||||
"cpython",
|
||||
"criterion",
|
||||
"dirs-next",
|
||||
"env_logger",
|
||||
"flame",
|
||||
@@ -1863,6 +2008,15 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.19"
|
||||
@@ -1915,6 +2069,16 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_cbor"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622"
|
||||
dependencies = [
|
||||
"half",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.118"
|
||||
@@ -2196,6 +2360,16 @@ version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cb4fa83bb73adf1c7219f4fe4bf3c0ac5635e4e51e070fad5df745a41bedfb8"
|
||||
|
||||
[[package]]
|
||||
name = "tinytemplate"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.1.0"
|
||||
@@ -2434,6 +2608,17 @@ version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8e76fae08f03f96e166d2dfda232190638c10e0383841252416f9cfe2ae60e6"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.9.0+wasi-snapshot-preview1"
|
||||
|
||||
+15
-4
@@ -14,10 +14,6 @@ members = [
|
||||
"derive", "jit", "parser", "vm", "vm/pylib-crate", "wasm/lib",
|
||||
]
|
||||
|
||||
[[bench]]
|
||||
name = "bench"
|
||||
path = "./benchmarks/bench.rs"
|
||||
|
||||
[features]
|
||||
default = ["threading", "pylib"]
|
||||
flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"]
|
||||
@@ -48,6 +44,15 @@ rustyline = "6.0"
|
||||
|
||||
[dev-dependencies]
|
||||
cpython = "0.5.0"
|
||||
criterion = "0.3"
|
||||
|
||||
[[bench]]
|
||||
name = "execution"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "microbenchmarks"
|
||||
harness = false
|
||||
|
||||
[[bin]]
|
||||
name = "rustpython"
|
||||
@@ -56,6 +61,11 @@ path = "src/main.rs"
|
||||
[profile.dev.package."*"]
|
||||
opt-level = 3
|
||||
|
||||
[profile.bench]
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
opt-level = 3
|
||||
|
||||
[patch.crates-io]
|
||||
# REDOX START, Uncommment when you want to compile/check with redoxer
|
||||
# # following patches are just waiting on a new version to be released to crates.io
|
||||
@@ -63,3 +73,4 @@ opt-level = 3
|
||||
# crossbeam-utils = { git = "https://github.com/crossbeam-rs/crossbeam" }
|
||||
# socket2 = { git = "https://github.com/alexcrichton/socket2-rs" }
|
||||
# REDOX END
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
# Benchmarking
|
||||
|
||||
These are some files to determine performance of rustpython.
|
||||
|
||||
## Usage
|
||||
|
||||
Running `cargo bench` from the root of the repository will start the benchmarks. Once done there will be a graphical
|
||||
report under `target/critierion/report/index.html` that you can use use to view the results.
|
||||
|
||||
To view Python tracebacks during benchmarks, run `RUST_BACKTRACE=1 cargo bench`. You can also bench against a
|
||||
specific installed Python version by running:
|
||||
|
||||
```shell
|
||||
$ PYTHON_SYS_EXECUTABLE=python3.7 cargo bench
|
||||
```
|
||||
|
||||
### Adding a benchmark
|
||||
|
||||
Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested
|
||||
in two ways:
|
||||
|
||||
1. The time to parse the file to AST
|
||||
2. The time it takes to execute the file
|
||||
|
||||
### Adding a micro benchmark
|
||||
|
||||
Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has
|
||||
two sections:
|
||||
1. Optional setup code
|
||||
2. The code to be benchmarked
|
||||
|
||||
These two sections are delimited by `# ---`. For example:
|
||||
|
||||
```python
|
||||
a_list = [1,2,3]
|
||||
|
||||
# ---
|
||||
|
||||
len(a_list)
|
||||
```
|
||||
|
||||
Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If
|
||||
present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For
|
||||
example:
|
||||
|
||||
```python
|
||||
obj = [i for i in range(ITERATIONS)]
|
||||
```
|
||||
|
||||
`ITERATIONS` can appear in both the setup code and the benchmark code.
|
||||
|
||||
## MacOS setup
|
||||
|
||||
On MacOS you will need to add the following to a `.cargo/config` file:
|
||||
|
||||
```toml
|
||||
[target.x86_64-apple-darwin]
|
||||
rustflags = [
|
||||
"-C", "link-arg=-undefined",
|
||||
"-C", "link-arg=dynamic_lookup",
|
||||
]
|
||||
```
|
||||
|
||||
## Benchmark source
|
||||
|
||||
- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html
|
||||
File renamed without changes.
@@ -1,22 +1,11 @@
|
||||
|
||||
# The Computer Language Benchmarks Game
|
||||
# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/
|
||||
# http://benchmarksgame.alioth.debian.org/
|
||||
#
|
||||
# originally by Kevin Carson
|
||||
# contributed by Kevin Carson
|
||||
# modified by Tupteq, Fredrik Johansson, and Daniel Nanz
|
||||
# modified by Maciej Fijalkowski
|
||||
# 2to3
|
||||
# modified by Andriy Misyura
|
||||
|
||||
from math import sqrt
|
||||
|
||||
def combinations(l):
|
||||
result = []
|
||||
for x in range(len(l) - 1):
|
||||
ls = l[x+1:]
|
||||
for y in ls:
|
||||
result.append((l[x][0],l[x][1],l[x][2],y[0],y[1],y[2]))
|
||||
return result
|
||||
import itertools
|
||||
import sys
|
||||
|
||||
PI = 3.14159265358979323
|
||||
SOLAR_MASS = 4 * PI * PI
|
||||
@@ -55,41 +44,44 @@ BODIES = {
|
||||
[2.68067772490389322e-03 * DAYS_PER_YEAR,
|
||||
1.62824170038242295e-03 * DAYS_PER_YEAR,
|
||||
-9.51592254519715870e-05 * DAYS_PER_YEAR],
|
||||
5.15138902046611451e-05 * SOLAR_MASS) }
|
||||
5.15138902046611451e-05 * SOLAR_MASS)}
|
||||
|
||||
SYSTEM = BODIES.values()
|
||||
PAIRS = list(itertools.combinations(SYSTEM, 2))
|
||||
|
||||
SYSTEM = tuple(BODIES.values())
|
||||
PAIRS = tuple(combinations(SYSTEM))
|
||||
|
||||
def advance(dt, n, bodies=SYSTEM, pairs=PAIRS):
|
||||
for i in range(n):
|
||||
for ([x1, y1, z1], v1, m1, [x2, y2, z2], v2, m2) in pairs:
|
||||
for (([x1, y1, z1], v1, m1),
|
||||
([x2, y2, z2], v2, m2)) in pairs:
|
||||
dx = x1 - x2
|
||||
dy = y1 - y2
|
||||
dz = z1 - z2
|
||||
dist = sqrt(dx * dx + dy * dy + dz * dz);
|
||||
mag = dt / (dist*dist*dist)
|
||||
mag = dt * ((dx * dx + dy * dy + dz * dz) ** (-1.5))
|
||||
b1m = m1 * mag
|
||||
b2m = m2 * mag
|
||||
v1[0] -= dx * b2m
|
||||
v1[1] -= dy * b2m
|
||||
v1[2] -= dz * b2m
|
||||
v2[2] += dz * b1m
|
||||
v2[1] += dy * b1m
|
||||
v2[0] += dx * b1m
|
||||
v2[1] += dy * b1m
|
||||
v2[2] += dz * b1m
|
||||
for (r, [vx, vy, vz], m) in bodies:
|
||||
r[0] += dt * vx
|
||||
r[1] += dt * vy
|
||||
r[2] += dt * vz
|
||||
|
||||
|
||||
def report_energy(bodies=SYSTEM, pairs=PAIRS, e=0.0):
|
||||
for ((x1, y1, z1), v1, m1, (x2, y2, z2), v2, m2) in pairs:
|
||||
for (((x1, y1, z1), v1, m1),
|
||||
((x2, y2, z2), v2, m2)) in pairs:
|
||||
dx = x1 - x2
|
||||
dy = y1 - y2
|
||||
dz = z1 - z2
|
||||
e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5)
|
||||
for (r, [vx, vy, vz], m) in bodies:
|
||||
e += m * (vx * vx + vy * vy + vz * vz) / 2.
|
||||
# print(f"{e}")
|
||||
|
||||
|
||||
def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0):
|
||||
for (r, [vx, vy, vz], m) in bodies:
|
||||
@@ -101,10 +93,12 @@ def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0):
|
||||
v[1] = py / m
|
||||
v[2] = pz / m
|
||||
|
||||
|
||||
def main(n, ref='sun'):
|
||||
offset_momentum(BODIES[ref])
|
||||
report_energy()
|
||||
advance(0.01, n)
|
||||
report_energy()
|
||||
|
||||
main(500)
|
||||
|
||||
main(int(500))
|
||||
@@ -41,10 +41,6 @@ Version History:
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
LOOPS = 50000
|
||||
|
||||
from time import time as clock
|
||||
|
||||
__version__ = "1.1.1"
|
||||
@@ -68,15 +64,6 @@ class Record:
|
||||
TRUE = 1
|
||||
FALSE = 0
|
||||
|
||||
def main(loops=LOOPS):
|
||||
benchtime, stones = pystones(loops)
|
||||
print("Pystone(%s) time for %d passes = %f" % \
|
||||
(__version__, loops, benchtime))
|
||||
print("This machine benchmarks at %f pystones/second" % stones)
|
||||
|
||||
|
||||
def pystones(loops=LOOPS):
|
||||
return Proc0(loops)
|
||||
|
||||
IntGlob = 0
|
||||
BoolGlob = FALSE
|
||||
@@ -87,7 +74,7 @@ Array2Glob = [x[:] for x in [Array1Glob]*51]
|
||||
PtrGlb = None
|
||||
PtrGlbNext = None
|
||||
|
||||
def Proc0(loops=LOOPS):
|
||||
def Proc0(loops):
|
||||
global IntGlob
|
||||
global BoolGlob
|
||||
global Char1Glob
|
||||
@@ -264,18 +251,10 @@ def Func3(EnumParIn):
|
||||
return FALSE
|
||||
|
||||
if __name__ == '__main__':
|
||||
import sys
|
||||
def error(msg):
|
||||
print(msg, end=' ', file=sys.stderr)
|
||||
print("usage: %s [number_of_loops]" % sys.argv[0], file=sys.stderr)
|
||||
sys.exit(100)
|
||||
nargs = len(sys.argv) - 1
|
||||
if nargs > 1:
|
||||
error("%d arguments are too many;" % nargs)
|
||||
elif nargs == 1:
|
||||
try: loops = int(sys.argv[1])
|
||||
except ValueError:
|
||||
error("Invalid argument %r;" % sys.argv[1])
|
||||
else:
|
||||
loops = LOOPS
|
||||
main(loops)
|
||||
if "LOOPS" not in globals():
|
||||
import sys
|
||||
if len(sys.argv) < 2:
|
||||
LOOPS = 50000
|
||||
else:
|
||||
LOOPS = int(sys.argv[1])
|
||||
Proc0(LOOPS)
|
||||
File renamed without changes.
@@ -0,0 +1,136 @@
|
||||
use criterion::measurement::WallTime;
|
||||
use criterion::{
|
||||
criterion_group, criterion_main, Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
|
||||
};
|
||||
use rustpython_compiler::Mode;
|
||||
use rustpython_parser::parser::parse_program;
|
||||
use rustpython_vm::pyobject::PyResult;
|
||||
use rustpython_vm::Interpreter;
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{fs, io};
|
||||
|
||||
fn bench_cpython_code(b: &mut Bencher, source: &str) {
|
||||
let gil = cpython::Python::acquire_gil();
|
||||
let python = gil.python();
|
||||
|
||||
b.iter(|| {
|
||||
let res: cpython::PyResult<()> = python.run(source, None, None);
|
||||
if let Err(e) = res {
|
||||
e.print(python);
|
||||
panic!("Error running source")
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn bench_rustpy_code(b: &mut Bencher, name: &str, source: &str) {
|
||||
// NOTE: Take long time.
|
||||
Interpreter::default().enter(|vm| {
|
||||
// Note: bench_cpython is both compiling and executing the code.
|
||||
// As such we compile the code in the benchmark loop as well.
|
||||
b.iter(|| {
|
||||
let code = vm.compile(source, Mode::Exec, name.to_owned()).unwrap();
|
||||
let scope = vm.new_scope_with_builtins();
|
||||
let res: PyResult = vm.run_code_obj(code.clone(), scope);
|
||||
vm.unwrap_pyresult(res);
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
pub fn benchmark_file_execution(
|
||||
group: &mut BenchmarkGroup<WallTime>,
|
||||
name: &str,
|
||||
contents: &String,
|
||||
) {
|
||||
group.bench_function(BenchmarkId::new(name, "cpython"), |b| {
|
||||
bench_cpython_code(b, &contents)
|
||||
});
|
||||
group.bench_function(BenchmarkId::new(name, "rustpython"), |b| {
|
||||
bench_rustpy_code(b, name, &contents)
|
||||
});
|
||||
}
|
||||
|
||||
pub fn benchmark_file_parsing(group: &mut BenchmarkGroup<WallTime>, name: &str, contents: &String) {
|
||||
group.throughput(Throughput::Bytes(contents.len() as u64));
|
||||
group.bench_function(BenchmarkId::new("rustpython", name), |b| {
|
||||
b.iter(|| parse_program(contents).unwrap())
|
||||
});
|
||||
group.bench_function(BenchmarkId::new("cpython", name), |b| {
|
||||
let gil = cpython::Python::acquire_gil();
|
||||
let python = gil.python();
|
||||
|
||||
let globals = None;
|
||||
let locals = cpython::PyDict::new(python);
|
||||
|
||||
locals.set_item(python, "SOURCE_CODE", &contents).unwrap();
|
||||
|
||||
let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")";
|
||||
b.iter(|| {
|
||||
let res: cpython::PyResult<cpython::PyObject> =
|
||||
python.eval(code, globals, Some(&locals));
|
||||
if let Err(e) = res {
|
||||
e.print(python);
|
||||
panic!("Error compiling source")
|
||||
}
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
pub fn benchmark_pystone(group: &mut BenchmarkGroup<WallTime>, contents: String) {
|
||||
// Default is 50_000. This takes a while, so reduce it to 30k.
|
||||
for idx in (10_000..=30_000).step_by(10_000) {
|
||||
let code_with_loops = format!("LOOPS = {}\n{}", idx, contents);
|
||||
let code_str = code_with_loops.as_str();
|
||||
|
||||
group.throughput(Throughput::Elements(idx as u64));
|
||||
group.bench_function(BenchmarkId::new("cpython", idx), |b| {
|
||||
bench_cpython_code(b, code_str)
|
||||
});
|
||||
group.bench_function(BenchmarkId::new("rustpython", idx), |b| {
|
||||
bench_rustpy_code(b, "pystone", code_str)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn criterion_benchmark(c: &mut Criterion) {
|
||||
let benchmark_dir = Path::new("./benches/benchmarks/");
|
||||
let dirs: Vec<fs::DirEntry> = benchmark_dir
|
||||
.read_dir()
|
||||
.unwrap()
|
||||
.collect::<io::Result<_>>()
|
||||
.unwrap();
|
||||
let paths: Vec<PathBuf> = dirs.iter().map(|p| p.path()).collect();
|
||||
|
||||
let mut name_to_contents: HashMap<String, String> = paths
|
||||
.into_iter()
|
||||
.map(|p| {
|
||||
let name = p.file_name().unwrap().to_os_string();
|
||||
let contents = fs::read_to_string(p).unwrap();
|
||||
(name.into_string().unwrap(), contents)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Benchmark parsing
|
||||
let mut parse_group = c.benchmark_group("parse_to_ast");
|
||||
for (name, contents) in name_to_contents.iter() {
|
||||
benchmark_file_parsing(&mut parse_group, name, contents);
|
||||
}
|
||||
parse_group.finish();
|
||||
|
||||
// Benchmark PyStone
|
||||
if let Some(pystone_contents) = name_to_contents.remove("pystone.py") {
|
||||
let mut pystone_group = c.benchmark_group("pystone");
|
||||
benchmark_pystone(&mut pystone_group, pystone_contents);
|
||||
pystone_group.finish();
|
||||
}
|
||||
|
||||
// Benchmark execution
|
||||
let mut execution_group = c.benchmark_group("execution");
|
||||
for (name, contents) in name_to_contents.iter() {
|
||||
benchmark_file_execution(&mut execution_group, name, contents);
|
||||
}
|
||||
execution_group.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, criterion_benchmark);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,169 @@
|
||||
use cpython::Python;
|
||||
use criterion::measurement::WallTime;
|
||||
use criterion::{
|
||||
criterion_group, criterion_main, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput,
|
||||
};
|
||||
use rustpython_compiler::Mode;
|
||||
use rustpython_vm::pyobject::ItemProtocol;
|
||||
use rustpython_vm::pyobject::PyResult;
|
||||
use rustpython_vm::{InitParameter, Interpreter, PySettings};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{fs, io};
|
||||
|
||||
pub struct MicroBenchmark {
|
||||
name: String,
|
||||
setup: String,
|
||||
code: String,
|
||||
iterate: bool,
|
||||
}
|
||||
|
||||
fn bench_cpython_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmark) {
|
||||
let gil = cpython::Python::acquire_gil();
|
||||
let python = gil.python();
|
||||
|
||||
let bench_func = |(python, code): (Python, String)| {
|
||||
let res: cpython::PyResult<()> = python.run(&code, None, None);
|
||||
if let Err(e) = res {
|
||||
e.print(python);
|
||||
panic!("Error running microbenchmark")
|
||||
}
|
||||
};
|
||||
|
||||
let bench_setup = |iterations| {
|
||||
let code = if let Some(idx) = iterations {
|
||||
// We can't easily modify the locals when running cPython. So we just add the
|
||||
// loop iterations at the top of the code...
|
||||
format!("ITERATIONS = {}\n{}", idx, bench.code)
|
||||
} else {
|
||||
(&bench.code).to_string()
|
||||
};
|
||||
|
||||
let res: cpython::PyResult<()> = python.run(&bench.setup, None, None);
|
||||
if let Err(e) = res {
|
||||
e.print(python);
|
||||
panic!("Error running microbenchmark setup code")
|
||||
}
|
||||
(python, code)
|
||||
};
|
||||
|
||||
if bench.iterate {
|
||||
for idx in (100..=1_000).step_by(200) {
|
||||
group.throughput(Throughput::Elements(idx as u64));
|
||||
group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| {
|
||||
b.iter_batched(
|
||||
|| bench_setup(Some(*idx)),
|
||||
bench_func,
|
||||
BatchSize::PerIteration,
|
||||
);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
group.bench_function(BenchmarkId::new("cpython", &bench.name), move |b| {
|
||||
b.iter_batched(|| bench_setup(None), bench_func, BatchSize::PerIteration);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
fn bench_rustpy_code(group: &mut BenchmarkGroup<WallTime>, bench: &MicroBenchmark) {
|
||||
let mut settings = PySettings::default();
|
||||
settings.path_list.push("Lib/".to_string());
|
||||
settings.dont_write_bytecode = true;
|
||||
settings.no_user_site = true;
|
||||
|
||||
Interpreter::new(settings, InitParameter::External).enter(|vm| {
|
||||
let setup_code = vm
|
||||
.compile(&bench.setup, Mode::Exec, bench.name.to_owned())
|
||||
.expect("Error compiling setup code");
|
||||
let bench_code = vm
|
||||
.compile(&bench.code, Mode::Exec, bench.name.to_owned())
|
||||
.expect("Error compiling bench code");
|
||||
|
||||
let bench_func = |(scope, bench_code)| {
|
||||
let res: PyResult = vm.run_code_obj(bench_code, scope);
|
||||
vm.unwrap_pyresult(res);
|
||||
};
|
||||
|
||||
let bench_setup = |iterations| {
|
||||
let scope = vm.new_scope_with_builtins();
|
||||
if let Some(idx) = iterations {
|
||||
scope
|
||||
.locals
|
||||
.set_item(vm.ctx.new_str("ITERATIONS"), vm.ctx.new_int(idx), vm)
|
||||
.expect("Error adding ITERATIONS local variable");
|
||||
}
|
||||
let setup_result = vm.run_code_obj(setup_code.clone(), scope.clone());
|
||||
vm.unwrap_pyresult(setup_result);
|
||||
(scope, bench_code.clone())
|
||||
};
|
||||
|
||||
if bench.iterate {
|
||||
for idx in (100..=1_000).step_by(200) {
|
||||
group.throughput(Throughput::Elements(idx as u64));
|
||||
group.bench_with_input(
|
||||
BenchmarkId::new("rustpython", &bench.name),
|
||||
&idx,
|
||||
|b, idx| {
|
||||
b.iter_batched(
|
||||
|| bench_setup(Some(*idx)),
|
||||
bench_func,
|
||||
BatchSize::PerIteration,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
} else {
|
||||
group.bench_function(BenchmarkId::new("rustpython", &bench.name), move |b| {
|
||||
b.iter_batched(|| bench_setup(None), bench_func, BatchSize::PerIteration);
|
||||
});
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run_micro_benchmark(c: &mut Criterion, benchmark: MicroBenchmark) {
|
||||
let mut group = c.benchmark_group("microbenchmarks");
|
||||
|
||||
bench_cpython_code(&mut group, &benchmark);
|
||||
bench_rustpy_code(&mut group, &benchmark);
|
||||
|
||||
group.finish();
|
||||
}
|
||||
|
||||
pub fn criterion_benchmark(c: &mut Criterion) {
|
||||
let benchmark_dir = Path::new("./benches/microbenchmarks/");
|
||||
let dirs: Vec<fs::DirEntry> = benchmark_dir
|
||||
.read_dir()
|
||||
.unwrap()
|
||||
.collect::<io::Result<_>>()
|
||||
.unwrap();
|
||||
let paths: Vec<PathBuf> = dirs.iter().map(|p| p.path()).collect();
|
||||
|
||||
let benchmarks: Vec<MicroBenchmark> = paths
|
||||
.into_iter()
|
||||
.map(|p| {
|
||||
let name = p.file_name().unwrap().to_os_string();
|
||||
let contents = fs::read_to_string(p).unwrap();
|
||||
let iterate = contents.contains("ITERATIONS");
|
||||
|
||||
let (setup, code) = if contents.contains("# ---") {
|
||||
let split: Vec<&str> = contents.splitn(2, "# ---").collect();
|
||||
(split[0].to_string(), split[1].to_string())
|
||||
} else {
|
||||
("".to_string(), contents)
|
||||
};
|
||||
let name = name.into_string().unwrap();
|
||||
MicroBenchmark {
|
||||
name,
|
||||
setup,
|
||||
code,
|
||||
iterate,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
for benchmark in benchmarks {
|
||||
run_micro_benchmark(c, benchmark);
|
||||
}
|
||||
}
|
||||
|
||||
criterion_group!(benches, criterion_benchmark);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,3 @@
|
||||
total = 0
|
||||
for i in range(ITERATIONS):
|
||||
total += i
|
||||
@@ -0,0 +1,7 @@
|
||||
def add(a, b):
|
||||
a + b
|
||||
|
||||
|
||||
# ---
|
||||
|
||||
add(a=1, b=10)
|
||||
@@ -0,0 +1,7 @@
|
||||
def add(a, b):
|
||||
a + b
|
||||
|
||||
|
||||
# ---
|
||||
|
||||
add(1, 2)
|
||||
@@ -0,0 +1,12 @@
|
||||
class Foo:
|
||||
ABC = 1
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def bar(self):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def bar_2(cls):
|
||||
pass
|
||||
@@ -0,0 +1 @@
|
||||
obj = {i: i for i in range(ITERATIONS)}
|
||||
@@ -0,0 +1 @@
|
||||
obj = [i for i in range(ITERATIONS)]
|
||||
@@ -0,0 +1 @@
|
||||
obj = {i for i in range(ITERATIONS)}
|
||||
@@ -0,0 +1,7 @@
|
||||
class Foo:
|
||||
pass
|
||||
|
||||
|
||||
# ---
|
||||
|
||||
Foo()
|
||||
@@ -0,0 +1,2 @@
|
||||
class Foo:
|
||||
pass
|
||||
@@ -0,0 +1,2 @@
|
||||
def function():
|
||||
pass
|
||||
@@ -0,0 +1,13 @@
|
||||
from contextlib import contextmanager
|
||||
|
||||
@contextmanager
|
||||
def try_catch(*args, **kwargs):
|
||||
try:
|
||||
yield
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
# ---
|
||||
|
||||
with try_catch():
|
||||
raise RuntimeError()
|
||||
@@ -0,0 +1,7 @@
|
||||
try:
|
||||
try:
|
||||
raise ValueError()
|
||||
except ValueError as e:
|
||||
raise RuntimeError() from e
|
||||
except RuntimeError as e:
|
||||
pass
|
||||
@@ -0,0 +1,4 @@
|
||||
try:
|
||||
raise RuntimeError()
|
||||
except RuntimeError as e:
|
||||
pass
|
||||
@@ -0,0 +1,6 @@
|
||||
obj = []
|
||||
|
||||
# ---
|
||||
|
||||
for i in range(ITERATIONS):
|
||||
obj.append(i)
|
||||
@@ -0,0 +1,6 @@
|
||||
string = "a" * ITERATIONS
|
||||
|
||||
# ---
|
||||
|
||||
for char in string:
|
||||
pass
|
||||
File renamed without changes.
@@ -1,22 +0,0 @@
|
||||
# Benchmarking
|
||||
|
||||
These are some files to determine performance of rustpython.
|
||||
|
||||
## Usage
|
||||
|
||||
Install pytest and pytest-benchmark:
|
||||
|
||||
$ pip install pytest-benchmark
|
||||
|
||||
Then run:
|
||||
|
||||
$ pytest
|
||||
|
||||
You can also benchmark the Rust benchmarks by just running
|
||||
`cargo +nightly bench` from the root of the repository. Make sure you have Rust
|
||||
nightly installed, as the benchmarking parts of the standard library are still
|
||||
unstable.
|
||||
|
||||
## Benchmark source
|
||||
|
||||
- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html
|
||||
@@ -1,118 +0,0 @@
|
||||
#![feature(test)]
|
||||
|
||||
extern crate test;
|
||||
|
||||
use test::Bencher;
|
||||
|
||||
use rustpython_compiler::compile;
|
||||
use rustpython_vm::pyobject::PyResult;
|
||||
use rustpython_vm::Interpreter;
|
||||
|
||||
const MINIDOM: &str = include_str!("./benchmarks/minidom.py");
|
||||
const NBODY: &str = include_str!("./benchmarks/nbody.py");
|
||||
const MANDELBROT: &str = include_str!("./benchmarks/mandelbrot.py");
|
||||
const PYSTONE: &str = include_str!("./benchmarks/pystone.py");
|
||||
|
||||
#[bench]
|
||||
fn bench_tokenization(b: &mut Bencher) {
|
||||
use rustpython_parser::lexer::{make_tokenizer, Tok};
|
||||
|
||||
let source = MINIDOM;
|
||||
|
||||
b.bytes = source.len() as _;
|
||||
b.iter(|| {
|
||||
let lexer = make_tokenizer(source);
|
||||
for res in lexer {
|
||||
let _token: Tok = res.unwrap().1;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_rustpy_parse_to_ast(b: &mut Bencher) {
|
||||
use rustpython_parser::parser::parse_program;
|
||||
|
||||
let source = MINIDOM;
|
||||
|
||||
b.bytes = source.len() as _;
|
||||
b.iter(|| parse_program(source).unwrap())
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_cpython_parse_to_ast(b: &mut Bencher) {
|
||||
let source = MINIDOM;
|
||||
|
||||
let gil = cpython::Python::acquire_gil();
|
||||
let python = gil.python();
|
||||
|
||||
let globals = None;
|
||||
let locals = cpython::PyDict::new(python);
|
||||
|
||||
locals.set_item(python, "SOURCE_CODE", source).unwrap();
|
||||
|
||||
let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")";
|
||||
|
||||
b.bytes = source.len() as _;
|
||||
b.iter(|| {
|
||||
let res: cpython::PyResult<cpython::PyObject> = python.eval(code, globals, Some(&locals));
|
||||
assert!(res.is_ok());
|
||||
})
|
||||
}
|
||||
|
||||
fn bench_cpython(b: &mut Bencher, source: &str) {
|
||||
let gil = cpython::Python::acquire_gil();
|
||||
let python = gil.python();
|
||||
|
||||
let globals = None;
|
||||
let locals = None;
|
||||
|
||||
b.iter(|| {
|
||||
let res: cpython::PyResult<()> = python.run(source, globals, locals);
|
||||
assert!(res.is_ok());
|
||||
})
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_cpython_nbody(b: &mut Bencher) {
|
||||
bench_cpython(b, NBODY)
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_cpython_mandelbrot(b: &mut Bencher) {
|
||||
bench_cpython(b, MANDELBROT)
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_cpython_pystone(b: &mut Bencher) {
|
||||
bench_cpython(b, PYSTONE)
|
||||
}
|
||||
|
||||
fn bench_rustpy(b: &mut Bencher, name: &str, source: &str) {
|
||||
// NOTE: Take long time.
|
||||
Interpreter::default().enter(|vm| {
|
||||
let code = vm
|
||||
.compile(source, compile::Mode::Exec, name.to_owned())
|
||||
.unwrap();
|
||||
|
||||
b.iter(|| {
|
||||
let scope = vm.new_scope_with_builtins();
|
||||
let res: PyResult = vm.run_code_obj(code.clone(), scope);
|
||||
vm.unwrap_pyresult(res);
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_rustpy_nbody(b: &mut Bencher) {
|
||||
bench_rustpy(b, "nbody.py", NBODY)
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_rustpy_mandelbrot(b: &mut Bencher) {
|
||||
bench_rustpy(b, "mandelbrot.py", MANDELBROT)
|
||||
}
|
||||
|
||||
#[bench]
|
||||
fn bench_rustpy_pystone(b: &mut Bencher) {
|
||||
bench_rustpy(b, "pystone.py", PYSTONE)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user