diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a29e303e6..7489c3d81 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -160,7 +160,7 @@ jobs: - name: install flake8 run: python -m pip install flake8 - name: run lint - run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib --select=E9,F63,F7,F82 --show-source --statistics + run: flake8 . --count --exclude=./.*,./Lib,./vm/Lib,./benches/ --select=E9,F63,F7,F82 --show-source --statistics miri: name: Run tests under miri runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index 270758d50..32d95e9cd 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ flame.txt flamescope.json /wapm.lock /wapm_packages +/.cargo/config diff --git a/Cargo.lock b/Cargo.lock index fa604124c..a2379c202 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,6 +243,15 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "cast" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b9434b9a5aa1450faa3f9cb14ea0e8c53bb5d2b3c1bfd1ab4fc03e9f33fbfb0" +dependencies = [ + "rustc_version", +] + [[package]] name = "cc" version = "1.0.66" @@ -305,6 +314,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "const_fn" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd51eab21ab4fd6a3bf889e2d0958c0a6e3a61ad04260325e919e652a2a62826" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -454,6 +469,77 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "criterion" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70daa7ceec6cf143990669a04c7df13391d55fb27bd4079d252fca774ba244d8" +dependencies = [ + "atty", + "cast", + "clap", + "criterion-plot", + "csv", + "itertools", + "lazy_static 1.4.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_cbor", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e022feadec601fba1649cfa83586381a4ad31c6bf3a9ab7d408118b05dd9889d" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d" +dependencies = [ + "cfg-if 1.0.0", + "const_fn", + "crossbeam-utils", + "lazy_static 1.4.0", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.1" @@ -802,6 +888,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "half" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36fab90f82edc3c747f9d438e06cf0a491055896f2a279638bb5beed6c40177" + [[package]] name = "hashbrown" version = "0.9.1" @@ -1093,6 +1185,15 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.4.3" @@ -1233,6 +1334,12 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" +[[package]] +name = "oorandom" +version = "11.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" + [[package]] name = "opaque-debug" version = "0.2.3" @@ -1351,6 +1458,18 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +[[package]] +name = "plotters" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d1685fbe7beba33de0330629da9d955ac75bd54f33d7b79f9a895590124f6bb" +dependencies = [ + "js-sys", + "num-traits", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "pmutil" version = "0.5.3" @@ -1475,6 +1594,31 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static 1.4.0", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.1.57" @@ -1595,6 +1739,7 @@ dependencies = [ "cfg-if 0.1.10", "clap", "cpython", + "criterion", "dirs-next", "env_logger", "flame", @@ -1863,6 +2008,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.19" @@ -1915,6 +2069,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "serde_cbor" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +dependencies = [ + "half", + "serde", +] + [[package]] name = "serde_derive" version = "1.0.118" @@ -2196,6 +2360,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cb4fa83bb73adf1c7219f4fe4bf3c0ac5635e4e51e070fad5df745a41bedfb8" +[[package]] +name = "tinytemplate" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d3dc76004a03cec1c5932bca4cdc2e39aaa798e3f82363dd94f9adf6098c12f" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.1.0" @@ -2434,6 +2608,17 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8e76fae08f03f96e166d2dfda232190638c10e0383841252416f9cfe2ae60e6" +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "wasi" version = "0.9.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 744ddd336..9f6c50390 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,10 +14,6 @@ members = [ "derive", "jit", "parser", "vm", "vm/pylib-crate", "wasm/lib", ] -[[bench]] -name = "bench" -path = "./benchmarks/bench.rs" - [features] default = ["threading", "pylib"] flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] @@ -48,6 +44,15 @@ rustyline = "6.0" [dev-dependencies] cpython = "0.5.0" +criterion = "0.3" + +[[bench]] +name = "execution" +harness = false + +[[bench]] +name = "microbenchmarks" +harness = false [[bin]] name = "rustpython" @@ -56,6 +61,11 @@ path = "src/main.rs" [profile.dev.package."*"] opt-level = 3 +[profile.bench] +lto = true +codegen-units = 1 +opt-level = 3 + [patch.crates-io] # REDOX START, Uncommment when you want to compile/check with redoxer # # following patches are just waiting on a new version to be released to crates.io @@ -63,3 +73,4 @@ opt-level = 3 # crossbeam-utils = { git = "https://github.com/crossbeam-rs/crossbeam" } # socket2 = { git = "https://github.com/alexcrichton/socket2-rs" } # REDOX END + diff --git a/benches/README.md b/benches/README.md new file mode 100644 index 000000000..0a9a93651 --- /dev/null +++ b/benches/README.md @@ -0,0 +1,66 @@ +# Benchmarking + +These are some files to determine performance of rustpython. + +## Usage + +Running `cargo bench` from the root of the repository will start the benchmarks. Once done there will be a graphical +report under `target/critierion/report/index.html` that you can use use to view the results. + +To view Python tracebacks during benchmarks, run `RUST_BACKTRACE=1 cargo bench`. You can also bench against a +specific installed Python version by running: + +```shell +$ PYTHON_SYS_EXECUTABLE=python3.7 cargo bench +``` + +### Adding a benchmark + +Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested +in two ways: + +1. The time to parse the file to AST +2. The time it takes to execute the file + +### Adding a micro benchmark + +Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has +two sections: +1. Optional setup code +2. The code to be benchmarked + +These two sections are delimited by `# ---`. For example: + +```python +a_list = [1,2,3] + +# --- + +len(a_list) +``` + +Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If +present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For +example: + +```python +obj = [i for i in range(ITERATIONS)] +``` + +`ITERATIONS` can appear in both the setup code and the benchmark code. + +## MacOS setup + +On MacOS you will need to add the following to a `.cargo/config` file: + +```toml +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] +``` + +## Benchmark source + +- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html diff --git a/benchmarks/benchmarks/mandelbrot.py b/benches/benchmarks/mandelbrot.py similarity index 100% rename from benchmarks/benchmarks/mandelbrot.py rename to benches/benchmarks/mandelbrot.py diff --git a/benchmarks/benchmarks/nbody.py b/benches/benchmarks/nbody.py similarity index 78% rename from benchmarks/benchmarks/nbody.py rename to benches/benchmarks/nbody.py index b9ab142df..d87177d24 100644 --- a/benchmarks/benchmarks/nbody.py +++ b/benches/benchmarks/nbody.py @@ -1,22 +1,11 @@ - # The Computer Language Benchmarks Game -# https://salsa.debian.org/benchmarksgame-team/benchmarksgame/ +# http://benchmarksgame.alioth.debian.org/ # -# originally by Kevin Carson +# contributed by Kevin Carson # modified by Tupteq, Fredrik Johansson, and Daniel Nanz -# modified by Maciej Fijalkowski -# 2to3 -# modified by Andriy Misyura -from math import sqrt - -def combinations(l): - result = [] - for x in range(len(l) - 1): - ls = l[x+1:] - for y in ls: - result.append((l[x][0],l[x][1],l[x][2],y[0],y[1],y[2])) - return result +import itertools +import sys PI = 3.14159265358979323 SOLAR_MASS = 4 * PI * PI @@ -55,41 +44,44 @@ BODIES = { [2.68067772490389322e-03 * DAYS_PER_YEAR, 1.62824170038242295e-03 * DAYS_PER_YEAR, -9.51592254519715870e-05 * DAYS_PER_YEAR], - 5.15138902046611451e-05 * SOLAR_MASS) } + 5.15138902046611451e-05 * SOLAR_MASS)} + +SYSTEM = BODIES.values() +PAIRS = list(itertools.combinations(SYSTEM, 2)) -SYSTEM = tuple(BODIES.values()) -PAIRS = tuple(combinations(SYSTEM)) def advance(dt, n, bodies=SYSTEM, pairs=PAIRS): for i in range(n): - for ([x1, y1, z1], v1, m1, [x2, y2, z2], v2, m2) in pairs: + for (([x1, y1, z1], v1, m1), + ([x2, y2, z2], v2, m2)) in pairs: dx = x1 - x2 dy = y1 - y2 dz = z1 - z2 - dist = sqrt(dx * dx + dy * dy + dz * dz); - mag = dt / (dist*dist*dist) + mag = dt * ((dx * dx + dy * dy + dz * dz) ** (-1.5)) b1m = m1 * mag b2m = m2 * mag v1[0] -= dx * b2m v1[1] -= dy * b2m v1[2] -= dz * b2m - v2[2] += dz * b1m - v2[1] += dy * b1m v2[0] += dx * b1m + v2[1] += dy * b1m + v2[2] += dz * b1m for (r, [vx, vy, vz], m) in bodies: r[0] += dt * vx r[1] += dt * vy r[2] += dt * vz + def report_energy(bodies=SYSTEM, pairs=PAIRS, e=0.0): - for ((x1, y1, z1), v1, m1, (x2, y2, z2), v2, m2) in pairs: + for (((x1, y1, z1), v1, m1), + ((x2, y2, z2), v2, m2)) in pairs: dx = x1 - x2 dy = y1 - y2 dz = z1 - z2 e -= (m1 * m2) / ((dx * dx + dy * dy + dz * dz) ** 0.5) for (r, [vx, vy, vz], m) in bodies: e += m * (vx * vx + vy * vy + vz * vz) / 2. - # print(f"{e}") + def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0): for (r, [vx, vy, vz], m) in bodies: @@ -101,10 +93,12 @@ def offset_momentum(ref, bodies=SYSTEM, px=0.0, py=0.0, pz=0.0): v[1] = py / m v[2] = pz / m + def main(n, ref='sun'): offset_momentum(BODIES[ref]) report_energy() advance(0.01, n) report_energy() -main(500) + +main(int(500)) diff --git a/benchmarks/benchmarks/pystone.py b/benches/benchmarks/pystone.py similarity index 89% rename from benchmarks/benchmarks/pystone.py rename to benches/benchmarks/pystone.py index 7c122cf67..3faf675ae 100644 --- a/benchmarks/benchmarks/pystone.py +++ b/benches/benchmarks/pystone.py @@ -41,10 +41,6 @@ Version History: """ -from __future__ import print_function - -LOOPS = 50000 - from time import time as clock __version__ = "1.1.1" @@ -68,15 +64,6 @@ class Record: TRUE = 1 FALSE = 0 -def main(loops=LOOPS): - benchtime, stones = pystones(loops) - print("Pystone(%s) time for %d passes = %f" % \ - (__version__, loops, benchtime)) - print("This machine benchmarks at %f pystones/second" % stones) - - -def pystones(loops=LOOPS): - return Proc0(loops) IntGlob = 0 BoolGlob = FALSE @@ -87,7 +74,7 @@ Array2Glob = [x[:] for x in [Array1Glob]*51] PtrGlb = None PtrGlbNext = None -def Proc0(loops=LOOPS): +def Proc0(loops): global IntGlob global BoolGlob global Char1Glob @@ -264,18 +251,10 @@ def Func3(EnumParIn): return FALSE if __name__ == '__main__': - import sys - def error(msg): - print(msg, end=' ', file=sys.stderr) - print("usage: %s [number_of_loops]" % sys.argv[0], file=sys.stderr) - sys.exit(100) - nargs = len(sys.argv) - 1 - if nargs > 1: - error("%d arguments are too many;" % nargs) - elif nargs == 1: - try: loops = int(sys.argv[1]) - except ValueError: - error("Invalid argument %r;" % sys.argv[1]) - else: - loops = LOOPS - main(loops) + if "LOOPS" not in globals(): + import sys + if len(sys.argv) < 2: + LOOPS = 50000 + else: + LOOPS = int(sys.argv[1]) + Proc0(LOOPS) diff --git a/benchmarks/benchmarks/strings.py b/benches/benchmarks/strings.py similarity index 100% rename from benchmarks/benchmarks/strings.py rename to benches/benchmarks/strings.py diff --git a/benches/execution.rs b/benches/execution.rs new file mode 100644 index 000000000..39c7ed6b7 --- /dev/null +++ b/benches/execution.rs @@ -0,0 +1,136 @@ +use criterion::measurement::WallTime; +use criterion::{ + criterion_group, criterion_main, Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, +}; +use rustpython_compiler::Mode; +use rustpython_parser::parser::parse_program; +use rustpython_vm::pyobject::PyResult; +use rustpython_vm::Interpreter; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::{fs, io}; + +fn bench_cpython_code(b: &mut Bencher, source: &str) { + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + b.iter(|| { + let res: cpython::PyResult<()> = python.run(source, None, None); + if let Err(e) = res { + e.print(python); + panic!("Error running source") + } + }); +} + +fn bench_rustpy_code(b: &mut Bencher, name: &str, source: &str) { + // NOTE: Take long time. + Interpreter::default().enter(|vm| { + // Note: bench_cpython is both compiling and executing the code. + // As such we compile the code in the benchmark loop as well. + b.iter(|| { + let code = vm.compile(source, Mode::Exec, name.to_owned()).unwrap(); + let scope = vm.new_scope_with_builtins(); + let res: PyResult = vm.run_code_obj(code.clone(), scope); + vm.unwrap_pyresult(res); + }) + }) +} + +pub fn benchmark_file_execution( + group: &mut BenchmarkGroup, + name: &str, + contents: &String, +) { + group.bench_function(BenchmarkId::new(name, "cpython"), |b| { + bench_cpython_code(b, &contents) + }); + group.bench_function(BenchmarkId::new(name, "rustpython"), |b| { + bench_rustpy_code(b, name, &contents) + }); +} + +pub fn benchmark_file_parsing(group: &mut BenchmarkGroup, name: &str, contents: &String) { + group.throughput(Throughput::Bytes(contents.len() as u64)); + group.bench_function(BenchmarkId::new("rustpython", name), |b| { + b.iter(|| parse_program(contents).unwrap()) + }); + group.bench_function(BenchmarkId::new("cpython", name), |b| { + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let globals = None; + let locals = cpython::PyDict::new(python); + + locals.set_item(python, "SOURCE_CODE", &contents).unwrap(); + + let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")"; + b.iter(|| { + let res: cpython::PyResult = + python.eval(code, globals, Some(&locals)); + if let Err(e) = res { + e.print(python); + panic!("Error compiling source") + } + }) + }); +} + +pub fn benchmark_pystone(group: &mut BenchmarkGroup, contents: String) { + // Default is 50_000. This takes a while, so reduce it to 30k. + for idx in (10_000..=30_000).step_by(10_000) { + let code_with_loops = format!("LOOPS = {}\n{}", idx, contents); + let code_str = code_with_loops.as_str(); + + group.throughput(Throughput::Elements(idx as u64)); + group.bench_function(BenchmarkId::new("cpython", idx), |b| { + bench_cpython_code(b, code_str) + }); + group.bench_function(BenchmarkId::new("rustpython", idx), |b| { + bench_rustpy_code(b, "pystone", code_str) + }); + } +} + +pub fn criterion_benchmark(c: &mut Criterion) { + let benchmark_dir = Path::new("./benches/benchmarks/"); + let dirs: Vec = benchmark_dir + .read_dir() + .unwrap() + .collect::>() + .unwrap(); + let paths: Vec = dirs.iter().map(|p| p.path()).collect(); + + let mut name_to_contents: HashMap = paths + .into_iter() + .map(|p| { + let name = p.file_name().unwrap().to_os_string(); + let contents = fs::read_to_string(p).unwrap(); + (name.into_string().unwrap(), contents) + }) + .collect(); + + // Benchmark parsing + let mut parse_group = c.benchmark_group("parse_to_ast"); + for (name, contents) in name_to_contents.iter() { + benchmark_file_parsing(&mut parse_group, name, contents); + } + parse_group.finish(); + + // Benchmark PyStone + if let Some(pystone_contents) = name_to_contents.remove("pystone.py") { + let mut pystone_group = c.benchmark_group("pystone"); + benchmark_pystone(&mut pystone_group, pystone_contents); + pystone_group.finish(); + } + + // Benchmark execution + let mut execution_group = c.benchmark_group("execution"); + for (name, contents) in name_to_contents.iter() { + benchmark_file_execution(&mut execution_group, name, contents); + } + execution_group.finish(); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs new file mode 100644 index 000000000..e37265681 --- /dev/null +++ b/benches/microbenchmarks.rs @@ -0,0 +1,169 @@ +use cpython::Python; +use criterion::measurement::WallTime; +use criterion::{ + criterion_group, criterion_main, BatchSize, BenchmarkGroup, BenchmarkId, Criterion, Throughput, +}; +use rustpython_compiler::Mode; +use rustpython_vm::pyobject::ItemProtocol; +use rustpython_vm::pyobject::PyResult; +use rustpython_vm::{InitParameter, Interpreter, PySettings}; +use std::path::{Path, PathBuf}; +use std::{fs, io}; + +pub struct MicroBenchmark { + name: String, + setup: String, + code: String, + iterate: bool, +} + +fn bench_cpython_code(group: &mut BenchmarkGroup, bench: &MicroBenchmark) { + let gil = cpython::Python::acquire_gil(); + let python = gil.python(); + + let bench_func = |(python, code): (Python, String)| { + let res: cpython::PyResult<()> = python.run(&code, None, None); + if let Err(e) = res { + e.print(python); + panic!("Error running microbenchmark") + } + }; + + let bench_setup = |iterations| { + let code = if let Some(idx) = iterations { + // We can't easily modify the locals when running cPython. So we just add the + // loop iterations at the top of the code... + format!("ITERATIONS = {}\n{}", idx, bench.code) + } else { + (&bench.code).to_string() + }; + + let res: cpython::PyResult<()> = python.run(&bench.setup, None, None); + if let Err(e) = res { + e.print(python); + panic!("Error running microbenchmark setup code") + } + (python, code) + }; + + if bench.iterate { + for idx in (100..=1_000).step_by(200) { + group.throughput(Throughput::Elements(idx as u64)); + group.bench_with_input(BenchmarkId::new("cpython", &bench.name), &idx, |b, idx| { + b.iter_batched( + || bench_setup(Some(*idx)), + bench_func, + BatchSize::PerIteration, + ); + }); + } + } else { + group.bench_function(BenchmarkId::new("cpython", &bench.name), move |b| { + b.iter_batched(|| bench_setup(None), bench_func, BatchSize::PerIteration); + }); + } +} + +fn bench_rustpy_code(group: &mut BenchmarkGroup, bench: &MicroBenchmark) { + let mut settings = PySettings::default(); + settings.path_list.push("Lib/".to_string()); + settings.dont_write_bytecode = true; + settings.no_user_site = true; + + Interpreter::new(settings, InitParameter::External).enter(|vm| { + let setup_code = vm + .compile(&bench.setup, Mode::Exec, bench.name.to_owned()) + .expect("Error compiling setup code"); + let bench_code = vm + .compile(&bench.code, Mode::Exec, bench.name.to_owned()) + .expect("Error compiling bench code"); + + let bench_func = |(scope, bench_code)| { + let res: PyResult = vm.run_code_obj(bench_code, scope); + vm.unwrap_pyresult(res); + }; + + let bench_setup = |iterations| { + let scope = vm.new_scope_with_builtins(); + if let Some(idx) = iterations { + scope + .locals + .set_item(vm.ctx.new_str("ITERATIONS"), vm.ctx.new_int(idx), vm) + .expect("Error adding ITERATIONS local variable"); + } + let setup_result = vm.run_code_obj(setup_code.clone(), scope.clone()); + vm.unwrap_pyresult(setup_result); + (scope, bench_code.clone()) + }; + + if bench.iterate { + for idx in (100..=1_000).step_by(200) { + group.throughput(Throughput::Elements(idx as u64)); + group.bench_with_input( + BenchmarkId::new("rustpython", &bench.name), + &idx, + |b, idx| { + b.iter_batched( + || bench_setup(Some(*idx)), + bench_func, + BatchSize::PerIteration, + ); + }, + ); + } + } else { + group.bench_function(BenchmarkId::new("rustpython", &bench.name), move |b| { + b.iter_batched(|| bench_setup(None), bench_func, BatchSize::PerIteration); + }); + } + }) +} + +pub fn run_micro_benchmark(c: &mut Criterion, benchmark: MicroBenchmark) { + let mut group = c.benchmark_group("microbenchmarks"); + + bench_cpython_code(&mut group, &benchmark); + bench_rustpy_code(&mut group, &benchmark); + + group.finish(); +} + +pub fn criterion_benchmark(c: &mut Criterion) { + let benchmark_dir = Path::new("./benches/microbenchmarks/"); + let dirs: Vec = benchmark_dir + .read_dir() + .unwrap() + .collect::>() + .unwrap(); + let paths: Vec = dirs.iter().map(|p| p.path()).collect(); + + let benchmarks: Vec = paths + .into_iter() + .map(|p| { + let name = p.file_name().unwrap().to_os_string(); + let contents = fs::read_to_string(p).unwrap(); + let iterate = contents.contains("ITERATIONS"); + + let (setup, code) = if contents.contains("# ---") { + let split: Vec<&str> = contents.splitn(2, "# ---").collect(); + (split[0].to_string(), split[1].to_string()) + } else { + ("".to_string(), contents) + }; + let name = name.into_string().unwrap(); + MicroBenchmark { + name, + setup, + code, + iterate, + } + }) + .collect(); + + for benchmark in benchmarks { + run_micro_benchmark(c, benchmark); + } +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/benches/microbenchmarks/addition.py b/benches/microbenchmarks/addition.py new file mode 100644 index 000000000..7f45838c6 --- /dev/null +++ b/benches/microbenchmarks/addition.py @@ -0,0 +1,3 @@ +total = 0 +for i in range(ITERATIONS): + total += i diff --git a/benches/microbenchmarks/call_kwargs.py b/benches/microbenchmarks/call_kwargs.py new file mode 100644 index 000000000..0b607e7d4 --- /dev/null +++ b/benches/microbenchmarks/call_kwargs.py @@ -0,0 +1,7 @@ +def add(a, b): + a + b + + +# --- + +add(a=1, b=10) diff --git a/benches/microbenchmarks/call_simple.py b/benches/microbenchmarks/call_simple.py new file mode 100644 index 000000000..739720bdf --- /dev/null +++ b/benches/microbenchmarks/call_simple.py @@ -0,0 +1,7 @@ +def add(a, b): + a + b + + +# --- + +add(1, 2) diff --git a/benches/microbenchmarks/complex_class.py b/benches/microbenchmarks/complex_class.py new file mode 100644 index 000000000..1df4c4bf7 --- /dev/null +++ b/benches/microbenchmarks/complex_class.py @@ -0,0 +1,12 @@ +class Foo: + ABC = 1 + + def __init__(self): + super().__init__() + + def bar(self): + pass + + @classmethod + def bar_2(cls): + pass diff --git a/benches/microbenchmarks/comprehension_dict.py b/benches/microbenchmarks/comprehension_dict.py new file mode 100644 index 000000000..539a66c85 --- /dev/null +++ b/benches/microbenchmarks/comprehension_dict.py @@ -0,0 +1 @@ +obj = {i: i for i in range(ITERATIONS)} diff --git a/benches/microbenchmarks/comprehension_list.py b/benches/microbenchmarks/comprehension_list.py new file mode 100644 index 000000000..e44a1020a --- /dev/null +++ b/benches/microbenchmarks/comprehension_list.py @@ -0,0 +1 @@ +obj = [i for i in range(ITERATIONS)] diff --git a/benches/microbenchmarks/comprehension_set.py b/benches/microbenchmarks/comprehension_set.py new file mode 100644 index 000000000..d6fea2550 --- /dev/null +++ b/benches/microbenchmarks/comprehension_set.py @@ -0,0 +1 @@ +obj = {i for i in range(ITERATIONS)} diff --git a/benches/microbenchmarks/construct_object.py b/benches/microbenchmarks/construct_object.py new file mode 100644 index 000000000..885f809e2 --- /dev/null +++ b/benches/microbenchmarks/construct_object.py @@ -0,0 +1,7 @@ +class Foo: + pass + + +# --- + +Foo() diff --git a/benches/microbenchmarks/define_class.py b/benches/microbenchmarks/define_class.py new file mode 100644 index 000000000..100bc636d --- /dev/null +++ b/benches/microbenchmarks/define_class.py @@ -0,0 +1,2 @@ +class Foo: + pass diff --git a/benches/microbenchmarks/define_function.py b/benches/microbenchmarks/define_function.py new file mode 100644 index 000000000..e7f4b098f --- /dev/null +++ b/benches/microbenchmarks/define_function.py @@ -0,0 +1,2 @@ +def function(): + pass diff --git a/benches/microbenchmarks/exception_context.py b/benches/microbenchmarks/exception_context.py new file mode 100644 index 000000000..778df23f5 --- /dev/null +++ b/benches/microbenchmarks/exception_context.py @@ -0,0 +1,13 @@ +from contextlib import contextmanager + +@contextmanager +def try_catch(*args, **kwargs): + try: + yield + except RuntimeError: + pass + +# --- + +with try_catch(): + raise RuntimeError() diff --git a/benches/microbenchmarks/exception_nested.py b/benches/microbenchmarks/exception_nested.py new file mode 100644 index 000000000..b8d007441 --- /dev/null +++ b/benches/microbenchmarks/exception_nested.py @@ -0,0 +1,7 @@ +try: + try: + raise ValueError() + except ValueError as e: + raise RuntimeError() from e +except RuntimeError as e: + pass diff --git a/benches/microbenchmarks/exception_simple.py b/benches/microbenchmarks/exception_simple.py new file mode 100644 index 000000000..8c15e2857 --- /dev/null +++ b/benches/microbenchmarks/exception_simple.py @@ -0,0 +1,4 @@ +try: + raise RuntimeError() +except RuntimeError as e: + pass diff --git a/benches/microbenchmarks/loop_append.py b/benches/microbenchmarks/loop_append.py new file mode 100644 index 000000000..81c177863 --- /dev/null +++ b/benches/microbenchmarks/loop_append.py @@ -0,0 +1,6 @@ +obj = [] + +# --- + +for i in range(ITERATIONS): + obj.append(i) diff --git a/benches/microbenchmarks/loop_string.py b/benches/microbenchmarks/loop_string.py new file mode 100644 index 000000000..186fc95d3 --- /dev/null +++ b/benches/microbenchmarks/loop_string.py @@ -0,0 +1,6 @@ +string = "a" * ITERATIONS + +# --- + +for char in string: + pass diff --git a/benchmarks/test_benchmarks.py b/benches/test_benchmarks.py similarity index 100% rename from benchmarks/test_benchmarks.py rename to benches/test_benchmarks.py diff --git a/benchmarks/README.md b/benchmarks/README.md deleted file mode 100644 index dfb6bcb0b..000000000 --- a/benchmarks/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Benchmarking - -These are some files to determine performance of rustpython. - -## Usage - -Install pytest and pytest-benchmark: - - $ pip install pytest-benchmark - -Then run: - - $ pytest - -You can also benchmark the Rust benchmarks by just running -`cargo +nightly bench` from the root of the repository. Make sure you have Rust -nightly installed, as the benchmarking parts of the standard library are still -unstable. - -## Benchmark source - -- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html diff --git a/benchmarks/bench.rs b/benchmarks/bench.rs deleted file mode 100644 index 786f5388a..000000000 --- a/benchmarks/bench.rs +++ /dev/null @@ -1,118 +0,0 @@ -#![feature(test)] - -extern crate test; - -use test::Bencher; - -use rustpython_compiler::compile; -use rustpython_vm::pyobject::PyResult; -use rustpython_vm::Interpreter; - -const MINIDOM: &str = include_str!("./benchmarks/minidom.py"); -const NBODY: &str = include_str!("./benchmarks/nbody.py"); -const MANDELBROT: &str = include_str!("./benchmarks/mandelbrot.py"); -const PYSTONE: &str = include_str!("./benchmarks/pystone.py"); - -#[bench] -fn bench_tokenization(b: &mut Bencher) { - use rustpython_parser::lexer::{make_tokenizer, Tok}; - - let source = MINIDOM; - - b.bytes = source.len() as _; - b.iter(|| { - let lexer = make_tokenizer(source); - for res in lexer { - let _token: Tok = res.unwrap().1; - } - }) -} - -#[bench] -fn bench_rustpy_parse_to_ast(b: &mut Bencher) { - use rustpython_parser::parser::parse_program; - - let source = MINIDOM; - - b.bytes = source.len() as _; - b.iter(|| parse_program(source).unwrap()) -} - -#[bench] -fn bench_cpython_parse_to_ast(b: &mut Bencher) { - let source = MINIDOM; - - let gil = cpython::Python::acquire_gil(); - let python = gil.python(); - - let globals = None; - let locals = cpython::PyDict::new(python); - - locals.set_item(python, "SOURCE_CODE", source).unwrap(); - - let code = "compile(SOURCE_CODE, mode=\"exec\", filename=\"minidom.py\")"; - - b.bytes = source.len() as _; - b.iter(|| { - let res: cpython::PyResult = python.eval(code, globals, Some(&locals)); - assert!(res.is_ok()); - }) -} - -fn bench_cpython(b: &mut Bencher, source: &str) { - let gil = cpython::Python::acquire_gil(); - let python = gil.python(); - - let globals = None; - let locals = None; - - b.iter(|| { - let res: cpython::PyResult<()> = python.run(source, globals, locals); - assert!(res.is_ok()); - }) -} - -#[bench] -fn bench_cpython_nbody(b: &mut Bencher) { - bench_cpython(b, NBODY) -} - -#[bench] -fn bench_cpython_mandelbrot(b: &mut Bencher) { - bench_cpython(b, MANDELBROT) -} - -#[bench] -fn bench_cpython_pystone(b: &mut Bencher) { - bench_cpython(b, PYSTONE) -} - -fn bench_rustpy(b: &mut Bencher, name: &str, source: &str) { - // NOTE: Take long time. - Interpreter::default().enter(|vm| { - let code = vm - .compile(source, compile::Mode::Exec, name.to_owned()) - .unwrap(); - - b.iter(|| { - let scope = vm.new_scope_with_builtins(); - let res: PyResult = vm.run_code_obj(code.clone(), scope); - vm.unwrap_pyresult(res); - }) - }) -} - -#[bench] -fn bench_rustpy_nbody(b: &mut Bencher) { - bench_rustpy(b, "nbody.py", NBODY) -} - -#[bench] -fn bench_rustpy_mandelbrot(b: &mut Bencher) { - bench_rustpy(b, "mandelbrot.py", MANDELBROT) -} - -#[bench] -fn bench_rustpy_pystone(b: &mut Bencher) { - bench_rustpy(b, "pystone.py", PYSTONE) -} diff --git a/benchmarks/benchmarks/minidom.py b/benchmarks/benchmarks/minidom.py deleted file mode 100644 index 24957ea14..000000000 --- a/benchmarks/benchmarks/minidom.py +++ /dev/null @@ -1,1981 +0,0 @@ -"""Simple implementation of the Level 1 DOM. - -Namespaces and other minor Level 2 features are also supported. - -parse("foo.xml") - -parseString("") - -Todo: -===== - * convenience methods for getting elements and text. - * more testing - * bring some of the writer and linearizer code into conformance with this - interface - * SAX 2 namespaces -""" - -import io -import xml.dom - -from xml.dom import EMPTY_NAMESPACE, EMPTY_PREFIX, XMLNS_NAMESPACE, domreg -from xml.dom.minicompat import * -from xml.dom.xmlbuilder import DOMImplementationLS, DocumentLS - -# This is used by the ID-cache invalidation checks; the list isn't -# actually complete, since the nodes being checked will never be the -# DOCUMENT_NODE or DOCUMENT_FRAGMENT_NODE. (The node being checked is -# the node being added or removed, not the node being modified.) -# -_nodeTypes_with_children = (xml.dom.Node.ELEMENT_NODE, - xml.dom.Node.ENTITY_REFERENCE_NODE) - - -class Node(xml.dom.Node): - namespaceURI = None # this is non-null only for elements and attributes - parentNode = None - ownerDocument = None - nextSibling = None - previousSibling = None - - prefix = EMPTY_PREFIX # non-null only for NS elements and attributes - - def __bool__(self): - return True - - def toxml(self, encoding=None): - return self.toprettyxml("", "", encoding) - - def toprettyxml(self, indent="\t", newl="\n", encoding=None): - if encoding is None: - writer = io.StringIO() - else: - writer = io.TextIOWrapper(io.BytesIO(), - encoding=encoding, - errors="xmlcharrefreplace", - newline='\n') - if self.nodeType == Node.DOCUMENT_NODE: - # Can pass encoding only to document, to put it into XML header - self.writexml(writer, "", indent, newl, encoding) - else: - self.writexml(writer, "", indent, newl) - if encoding is None: - return writer.getvalue() - else: - return writer.detach().getvalue() - - def hasChildNodes(self): - return bool(self.childNodes) - - def _get_childNodes(self): - return self.childNodes - - def _get_firstChild(self): - if self.childNodes: - return self.childNodes[0] - - def _get_lastChild(self): - if self.childNodes: - return self.childNodes[-1] - - def insertBefore(self, newChild, refChild): - if newChild.nodeType == self.DOCUMENT_FRAGMENT_NODE: - for c in tuple(newChild.childNodes): - self.insertBefore(c, refChild) - ### The DOM does not clearly specify what to return in this case - return newChild - if newChild.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(newChild), repr(self))) - if newChild.parentNode is not None: - newChild.parentNode.removeChild(newChild) - if refChild is None: - self.appendChild(newChild) - else: - try: - index = self.childNodes.index(refChild) - except ValueError: - raise xml.dom.NotFoundErr() - if newChild.nodeType in _nodeTypes_with_children: - _clear_id_cache(self) - self.childNodes.insert(index, newChild) - newChild.nextSibling = refChild - refChild.previousSibling = newChild - if index: - node = self.childNodes[index-1] - node.nextSibling = newChild - newChild.previousSibling = node - else: - newChild.previousSibling = None - newChild.parentNode = self - return newChild - - def appendChild(self, node): - if node.nodeType == self.DOCUMENT_FRAGMENT_NODE: - for c in tuple(node.childNodes): - self.appendChild(c) - ### The DOM does not clearly specify what to return in this case - return node - if node.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(node), repr(self))) - elif node.nodeType in _nodeTypes_with_children: - _clear_id_cache(self) - if node.parentNode is not None: - node.parentNode.removeChild(node) - _append_child(self, node) - node.nextSibling = None - return node - - def replaceChild(self, newChild, oldChild): - if newChild.nodeType == self.DOCUMENT_FRAGMENT_NODE: - refChild = oldChild.nextSibling - self.removeChild(oldChild) - return self.insertBefore(newChild, refChild) - if newChild.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(newChild), repr(self))) - if newChild is oldChild: - return - if newChild.parentNode is not None: - newChild.parentNode.removeChild(newChild) - try: - index = self.childNodes.index(oldChild) - except ValueError: - raise xml.dom.NotFoundErr() - self.childNodes[index] = newChild - newChild.parentNode = self - oldChild.parentNode = None - if (newChild.nodeType in _nodeTypes_with_children - or oldChild.nodeType in _nodeTypes_with_children): - _clear_id_cache(self) - newChild.nextSibling = oldChild.nextSibling - newChild.previousSibling = oldChild.previousSibling - oldChild.nextSibling = None - oldChild.previousSibling = None - if newChild.previousSibling: - newChild.previousSibling.nextSibling = newChild - if newChild.nextSibling: - newChild.nextSibling.previousSibling = newChild - return oldChild - - def removeChild(self, oldChild): - try: - self.childNodes.remove(oldChild) - except ValueError: - raise xml.dom.NotFoundErr() - if oldChild.nextSibling is not None: - oldChild.nextSibling.previousSibling = oldChild.previousSibling - if oldChild.previousSibling is not None: - oldChild.previousSibling.nextSibling = oldChild.nextSibling - oldChild.nextSibling = oldChild.previousSibling = None - if oldChild.nodeType in _nodeTypes_with_children: - _clear_id_cache(self) - - oldChild.parentNode = None - return oldChild - - def normalize(self): - L = [] - for child in self.childNodes: - if child.nodeType == Node.TEXT_NODE: - if not child.data: - # empty text node; discard - if L: - L[-1].nextSibling = child.nextSibling - if child.nextSibling: - child.nextSibling.previousSibling = child.previousSibling - child.unlink() - elif L and L[-1].nodeType == child.nodeType: - # collapse text node - node = L[-1] - node.data = node.data + child.data - node.nextSibling = child.nextSibling - if child.nextSibling: - child.nextSibling.previousSibling = node - child.unlink() - else: - L.append(child) - else: - L.append(child) - if child.nodeType == Node.ELEMENT_NODE: - child.normalize() - self.childNodes[:] = L - - def cloneNode(self, deep): - return _clone_node(self, deep, self.ownerDocument or self) - - def isSupported(self, feature, version): - return self.ownerDocument.implementation.hasFeature(feature, version) - - def _get_localName(self): - # Overridden in Element and Attr where localName can be Non-Null - return None - - # Node interfaces from Level 3 (WD 9 April 2002) - - def isSameNode(self, other): - return self is other - - def getInterface(self, feature): - if self.isSupported(feature, None): - return self - else: - return None - - # The "user data" functions use a dictionary that is only present - # if some user data has been set, so be careful not to assume it - # exists. - - def getUserData(self, key): - try: - return self._user_data[key][0] - except (AttributeError, KeyError): - return None - - def setUserData(self, key, data, handler): - old = None - try: - d = self._user_data - except AttributeError: - d = {} - self._user_data = d - if key in d: - old = d[key][0] - if data is None: - # ignore handlers passed for None - handler = None - if old is not None: - del d[key] - else: - d[key] = (data, handler) - return old - - def _call_user_data_handler(self, operation, src, dst): - if hasattr(self, "_user_data"): - for key, (data, handler) in list(self._user_data.items()): - if handler is not None: - handler.handle(operation, key, data, src, dst) - - # minidom-specific API: - - def unlink(self): - self.parentNode = self.ownerDocument = None - if self.childNodes: - for child in self.childNodes: - child.unlink() - self.childNodes = NodeList() - self.previousSibling = None - self.nextSibling = None - - # A Node is its own context manager, to ensure that an unlink() call occurs. - # This is similar to how a file object works. - def __enter__(self): - return self - - def __exit__(self, et, ev, tb): - self.unlink() - -defproperty(Node, "firstChild", doc="First child node, or None.") -defproperty(Node, "lastChild", doc="Last child node, or None.") -defproperty(Node, "localName", doc="Namespace-local name of this node.") - - -def _append_child(self, node): - # fast path with less checks; usable by DOM builders if careful - childNodes = self.childNodes - if childNodes: - last = childNodes[-1] - node.previousSibling = last - last.nextSibling = node - childNodes.append(node) - node.parentNode = self - -def _in_document(node): - # return True iff node is part of a document tree - while node is not None: - if node.nodeType == Node.DOCUMENT_NODE: - return True - node = node.parentNode - return False - -def _write_data(writer, data): - "Writes datachars to writer." - if data: - data = data.replace("&", "&").replace("<", "<"). \ - replace("\"", """).replace(">", ">") - writer.write(data) - -def _get_elements_by_tagName_helper(parent, name, rc): - for node in parent.childNodes: - if node.nodeType == Node.ELEMENT_NODE and \ - (name == "*" or node.tagName == name): - rc.append(node) - _get_elements_by_tagName_helper(node, name, rc) - return rc - -def _get_elements_by_tagName_ns_helper(parent, nsURI, localName, rc): - for node in parent.childNodes: - if node.nodeType == Node.ELEMENT_NODE: - if ((localName == "*" or node.localName == localName) and - (nsURI == "*" or node.namespaceURI == nsURI)): - rc.append(node) - _get_elements_by_tagName_ns_helper(node, nsURI, localName, rc) - return rc - -class DocumentFragment(Node): - nodeType = Node.DOCUMENT_FRAGMENT_NODE - nodeName = "#document-fragment" - nodeValue = None - attributes = None - parentNode = None - _child_node_types = (Node.ELEMENT_NODE, - Node.TEXT_NODE, - Node.CDATA_SECTION_NODE, - Node.ENTITY_REFERENCE_NODE, - Node.PROCESSING_INSTRUCTION_NODE, - Node.COMMENT_NODE, - Node.NOTATION_NODE) - - def __init__(self): - self.childNodes = NodeList() - - -class Attr(Node): - __slots__=('_name', '_value', 'namespaceURI', - '_prefix', 'childNodes', '_localName', 'ownerDocument', 'ownerElement') - nodeType = Node.ATTRIBUTE_NODE - attributes = None - specified = False - _is_id = False - - _child_node_types = (Node.TEXT_NODE, Node.ENTITY_REFERENCE_NODE) - - def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, - prefix=None): - self.ownerElement = None - self._name = qName - self.namespaceURI = namespaceURI - self._prefix = prefix - self.childNodes = NodeList() - - # Add the single child node that represents the value of the attr - self.childNodes.append(Text()) - - # nodeValue and value are set elsewhere - - def _get_localName(self): - try: - return self._localName - except AttributeError: - return self.nodeName.split(":", 1)[-1] - - def _get_specified(self): - return self.specified - - def _get_name(self): - return self._name - - def _set_name(self, value): - self._name = value - if self.ownerElement is not None: - _clear_id_cache(self.ownerElement) - - nodeName = name = property(_get_name, _set_name) - - def _get_value(self): - return self._value - - def _set_value(self, value): - self._value = value - self.childNodes[0].data = value - if self.ownerElement is not None: - _clear_id_cache(self.ownerElement) - self.childNodes[0].data = value - - nodeValue = value = property(_get_value, _set_value) - - def _get_prefix(self): - return self._prefix - - def _set_prefix(self, prefix): - nsuri = self.namespaceURI - if prefix == "xmlns": - if nsuri and nsuri != XMLNS_NAMESPACE: - raise xml.dom.NamespaceErr( - "illegal use of 'xmlns' prefix for the wrong namespace") - self._prefix = prefix - if prefix is None: - newName = self.localName - else: - newName = "%s:%s" % (prefix, self.localName) - if self.ownerElement: - _clear_id_cache(self.ownerElement) - self.name = newName - - prefix = property(_get_prefix, _set_prefix) - - def unlink(self): - # This implementation does not call the base implementation - # since most of that is not needed, and the expense of the - # method call is not warranted. We duplicate the removal of - # children, but that's all we needed from the base class. - elem = self.ownerElement - if elem is not None: - del elem._attrs[self.nodeName] - del elem._attrsNS[(self.namespaceURI, self.localName)] - if self._is_id: - self._is_id = False - elem._magic_id_nodes -= 1 - self.ownerDocument._magic_id_count -= 1 - for child in self.childNodes: - child.unlink() - del self.childNodes[:] - - def _get_isId(self): - if self._is_id: - return True - doc = self.ownerDocument - elem = self.ownerElement - if doc is None or elem is None: - return False - - info = doc._get_elem_info(elem) - if info is None: - return False - if self.namespaceURI: - return info.isIdNS(self.namespaceURI, self.localName) - else: - return info.isId(self.nodeName) - - def _get_schemaType(self): - doc = self.ownerDocument - elem = self.ownerElement - if doc is None or elem is None: - return _no_type - - info = doc._get_elem_info(elem) - if info is None: - return _no_type - if self.namespaceURI: - return info.getAttributeTypeNS(self.namespaceURI, self.localName) - else: - return info.getAttributeType(self.nodeName) - -defproperty(Attr, "isId", doc="True if this attribute is an ID.") -defproperty(Attr, "localName", doc="Namespace-local name of this attribute.") -defproperty(Attr, "schemaType", doc="Schema type for this attribute.") - - -class NamedNodeMap(object): - """The attribute list is a transient interface to the underlying - dictionaries. Mutations here will change the underlying element's - dictionary. - - Ordering is imposed artificially and does not reflect the order of - attributes as found in an input document. - """ - - __slots__ = ('_attrs', '_attrsNS', '_ownerElement') - - def __init__(self, attrs, attrsNS, ownerElement): - self._attrs = attrs - self._attrsNS = attrsNS - self._ownerElement = ownerElement - - def _get_length(self): - return len(self._attrs) - - def item(self, index): - try: - return self[list(self._attrs.keys())[index]] - except IndexError: - return None - - def items(self): - L = [] - for node in self._attrs.values(): - L.append((node.nodeName, node.value)) - return L - - def itemsNS(self): - L = [] - for node in self._attrs.values(): - L.append(((node.namespaceURI, node.localName), node.value)) - return L - - def __contains__(self, key): - if isinstance(key, str): - return key in self._attrs - else: - return key in self._attrsNS - - def keys(self): - return self._attrs.keys() - - def keysNS(self): - return self._attrsNS.keys() - - def values(self): - return self._attrs.values() - - def get(self, name, value=None): - return self._attrs.get(name, value) - - __len__ = _get_length - - def _cmp(self, other): - if self._attrs is getattr(other, "_attrs", None): - return 0 - else: - return (id(self) > id(other)) - (id(self) < id(other)) - - def __eq__(self, other): - return self._cmp(other) == 0 - - def __ge__(self, other): - return self._cmp(other) >= 0 - - def __gt__(self, other): - return self._cmp(other) > 0 - - def __le__(self, other): - return self._cmp(other) <= 0 - - def __lt__(self, other): - return self._cmp(other) < 0 - - def __getitem__(self, attname_or_tuple): - if isinstance(attname_or_tuple, tuple): - return self._attrsNS[attname_or_tuple] - else: - return self._attrs[attname_or_tuple] - - # same as set - def __setitem__(self, attname, value): - if isinstance(value, str): - try: - node = self._attrs[attname] - except KeyError: - node = Attr(attname) - node.ownerDocument = self._ownerElement.ownerDocument - self.setNamedItem(node) - node.value = value - else: - if not isinstance(value, Attr): - raise TypeError("value must be a string or Attr object") - node = value - self.setNamedItem(node) - - def getNamedItem(self, name): - try: - return self._attrs[name] - except KeyError: - return None - - def getNamedItemNS(self, namespaceURI, localName): - try: - return self._attrsNS[(namespaceURI, localName)] - except KeyError: - return None - - def removeNamedItem(self, name): - n = self.getNamedItem(name) - if n is not None: - _clear_id_cache(self._ownerElement) - del self._attrs[n.nodeName] - del self._attrsNS[(n.namespaceURI, n.localName)] - if hasattr(n, 'ownerElement'): - n.ownerElement = None - return n - else: - raise xml.dom.NotFoundErr() - - def removeNamedItemNS(self, namespaceURI, localName): - n = self.getNamedItemNS(namespaceURI, localName) - if n is not None: - _clear_id_cache(self._ownerElement) - del self._attrsNS[(n.namespaceURI, n.localName)] - del self._attrs[n.nodeName] - if hasattr(n, 'ownerElement'): - n.ownerElement = None - return n - else: - raise xml.dom.NotFoundErr() - - def setNamedItem(self, node): - if not isinstance(node, Attr): - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(node), repr(self))) - old = self._attrs.get(node.name) - if old: - old.unlink() - self._attrs[node.name] = node - self._attrsNS[(node.namespaceURI, node.localName)] = node - node.ownerElement = self._ownerElement - _clear_id_cache(node.ownerElement) - return old - - def setNamedItemNS(self, node): - return self.setNamedItem(node) - - def __delitem__(self, attname_or_tuple): - node = self[attname_or_tuple] - _clear_id_cache(node.ownerElement) - node.unlink() - - def __getstate__(self): - return self._attrs, self._attrsNS, self._ownerElement - - def __setstate__(self, state): - self._attrs, self._attrsNS, self._ownerElement = state - -defproperty(NamedNodeMap, "length", - doc="Number of nodes in the NamedNodeMap.") - -AttributeList = NamedNodeMap - - -class TypeInfo(object): - __slots__ = 'namespace', 'name' - - def __init__(self, namespace, name): - self.namespace = namespace - self.name = name - - def __repr__(self): - if self.namespace: - return "<%s %r (from %r)>" % (self.__class__.__name__, self.name, - self.namespace) - else: - return "<%s %r>" % (self.__class__.__name__, self.name) - - def _get_name(self): - return self.name - - def _get_namespace(self): - return self.namespace - -_no_type = TypeInfo(None, None) - -class Element(Node): - __slots__=('ownerDocument', 'parentNode', 'tagName', 'nodeName', 'prefix', - 'namespaceURI', '_localName', 'childNodes', '_attrs', '_attrsNS', - 'nextSibling', 'previousSibling') - nodeType = Node.ELEMENT_NODE - nodeValue = None - schemaType = _no_type - - _magic_id_nodes = 0 - - _child_node_types = (Node.ELEMENT_NODE, - Node.PROCESSING_INSTRUCTION_NODE, - Node.COMMENT_NODE, - Node.TEXT_NODE, - Node.CDATA_SECTION_NODE, - Node.ENTITY_REFERENCE_NODE) - - def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None, - localName=None): - self.parentNode = None - self.tagName = self.nodeName = tagName - self.prefix = prefix - self.namespaceURI = namespaceURI - self.childNodes = NodeList() - self.nextSibling = self.previousSibling = None - - # Attribute dictionaries are lazily created - # attributes are double-indexed: - # tagName -> Attribute - # URI,localName -> Attribute - # in the future: consider lazy generation - # of attribute objects this is too tricky - # for now because of headaches with - # namespaces. - self._attrs = None - self._attrsNS = None - - def _ensure_attributes(self): - if self._attrs is None: - self._attrs = {} - self._attrsNS = {} - - def _get_localName(self): - try: - return self._localName - except AttributeError: - return self.tagName.split(":", 1)[-1] - - def _get_tagName(self): - return self.tagName - - def unlink(self): - if self._attrs is not None: - for attr in list(self._attrs.values()): - attr.unlink() - self._attrs = None - self._attrsNS = None - Node.unlink(self) - - def getAttribute(self, attname): - if self._attrs is None: - return "" - try: - return self._attrs[attname].value - except KeyError: - return "" - - def getAttributeNS(self, namespaceURI, localName): - if self._attrsNS is None: - return "" - try: - return self._attrsNS[(namespaceURI, localName)].value - except KeyError: - return "" - - def setAttribute(self, attname, value): - attr = self.getAttributeNode(attname) - if attr is None: - attr = Attr(attname) - attr.value = value # also sets nodeValue - attr.ownerDocument = self.ownerDocument - self.setAttributeNode(attr) - elif value != attr.value: - attr.value = value - if attr.isId: - _clear_id_cache(self) - - def setAttributeNS(self, namespaceURI, qualifiedName, value): - prefix, localname = _nssplit(qualifiedName) - attr = self.getAttributeNodeNS(namespaceURI, localname) - if attr is None: - attr = Attr(qualifiedName, namespaceURI, localname, prefix) - attr.value = value - attr.ownerDocument = self.ownerDocument - self.setAttributeNode(attr) - else: - if value != attr.value: - attr.value = value - if attr.isId: - _clear_id_cache(self) - if attr.prefix != prefix: - attr.prefix = prefix - attr.nodeName = qualifiedName - - def getAttributeNode(self, attrname): - if self._attrs is None: - return None - return self._attrs.get(attrname) - - def getAttributeNodeNS(self, namespaceURI, localName): - if self._attrsNS is None: - return None - return self._attrsNS.get((namespaceURI, localName)) - - def setAttributeNode(self, attr): - if attr.ownerElement not in (None, self): - raise xml.dom.InuseAttributeErr("attribute node already owned") - self._ensure_attributes() - old1 = self._attrs.get(attr.name, None) - if old1 is not None: - self.removeAttributeNode(old1) - old2 = self._attrsNS.get((attr.namespaceURI, attr.localName), None) - if old2 is not None and old2 is not old1: - self.removeAttributeNode(old2) - _set_attribute_node(self, attr) - - if old1 is not attr: - # It might have already been part of this node, in which case - # it doesn't represent a change, and should not be returned. - return old1 - if old2 is not attr: - return old2 - - setAttributeNodeNS = setAttributeNode - - def removeAttribute(self, name): - if self._attrsNS is None: - raise xml.dom.NotFoundErr() - try: - attr = self._attrs[name] - except KeyError: - raise xml.dom.NotFoundErr() - self.removeAttributeNode(attr) - - def removeAttributeNS(self, namespaceURI, localName): - if self._attrsNS is None: - raise xml.dom.NotFoundErr() - try: - attr = self._attrsNS[(namespaceURI, localName)] - except KeyError: - raise xml.dom.NotFoundErr() - self.removeAttributeNode(attr) - - def removeAttributeNode(self, node): - if node is None: - raise xml.dom.NotFoundErr() - try: - self._attrs[node.name] - except KeyError: - raise xml.dom.NotFoundErr() - _clear_id_cache(self) - node.unlink() - # Restore this since the node is still useful and otherwise - # unlinked - node.ownerDocument = self.ownerDocument - - removeAttributeNodeNS = removeAttributeNode - - def hasAttribute(self, name): - if self._attrs is None: - return False - return name in self._attrs - - def hasAttributeNS(self, namespaceURI, localName): - if self._attrsNS is None: - return False - return (namespaceURI, localName) in self._attrsNS - - def getElementsByTagName(self, name): - return _get_elements_by_tagName_helper(self, name, NodeList()) - - def getElementsByTagNameNS(self, namespaceURI, localName): - return _get_elements_by_tagName_ns_helper( - self, namespaceURI, localName, NodeList()) - - def __repr__(self): - return "" % (self.tagName, id(self)) - - def writexml(self, writer, indent="", addindent="", newl=""): - # indent = current indentation - # addindent = indentation to add to higher levels - # newl = newline string - writer.write(indent+"<" + self.tagName) - - attrs = self._get_attributes() - a_names = sorted(attrs.keys()) - - for a_name in a_names: - writer.write(" %s=\"" % a_name) - _write_data(writer, attrs[a_name].value) - writer.write("\"") - if self.childNodes: - writer.write(">") - if (len(self.childNodes) == 1 and - self.childNodes[0].nodeType == Node.TEXT_NODE): - self.childNodes[0].writexml(writer, '', '', '') - else: - writer.write(newl) - for node in self.childNodes: - node.writexml(writer, indent+addindent, addindent, newl) - writer.write(indent) - writer.write("%s" % (self.tagName, newl)) - else: - writer.write("/>%s"%(newl)) - - def _get_attributes(self): - self._ensure_attributes() - return NamedNodeMap(self._attrs, self._attrsNS, self) - - def hasAttributes(self): - if self._attrs: - return True - else: - return False - - # DOM Level 3 attributes, based on the 22 Oct 2002 draft - - def setIdAttribute(self, name): - idAttr = self.getAttributeNode(name) - self.setIdAttributeNode(idAttr) - - def setIdAttributeNS(self, namespaceURI, localName): - idAttr = self.getAttributeNodeNS(namespaceURI, localName) - self.setIdAttributeNode(idAttr) - - def setIdAttributeNode(self, idAttr): - if idAttr is None or not self.isSameNode(idAttr.ownerElement): - raise xml.dom.NotFoundErr() - if _get_containing_entref(self) is not None: - raise xml.dom.NoModificationAllowedErr() - if not idAttr._is_id: - idAttr._is_id = True - self._magic_id_nodes += 1 - self.ownerDocument._magic_id_count += 1 - _clear_id_cache(self) - -defproperty(Element, "attributes", - doc="NamedNodeMap of attributes on the element.") -defproperty(Element, "localName", - doc="Namespace-local name of this element.") - - -def _set_attribute_node(element, attr): - _clear_id_cache(element) - element._ensure_attributes() - element._attrs[attr.name] = attr - element._attrsNS[(attr.namespaceURI, attr.localName)] = attr - - # This creates a circular reference, but Element.unlink() - # breaks the cycle since the references to the attribute - # dictionaries are tossed. - attr.ownerElement = element - -class Childless: - """Mixin that makes childless-ness easy to implement and avoids - the complexity of the Node methods that deal with children. - """ - __slots__ = () - - attributes = None - childNodes = EmptyNodeList() - firstChild = None - lastChild = None - - def _get_firstChild(self): - return None - - def _get_lastChild(self): - return None - - def appendChild(self, node): - raise xml.dom.HierarchyRequestErr( - self.nodeName + " nodes cannot have children") - - def hasChildNodes(self): - return False - - def insertBefore(self, newChild, refChild): - raise xml.dom.HierarchyRequestErr( - self.nodeName + " nodes do not have children") - - def removeChild(self, oldChild): - raise xml.dom.NotFoundErr( - self.nodeName + " nodes do not have children") - - def normalize(self): - # For childless nodes, normalize() has nothing to do. - pass - - def replaceChild(self, newChild, oldChild): - raise xml.dom.HierarchyRequestErr( - self.nodeName + " nodes do not have children") - - -class ProcessingInstruction(Childless, Node): - nodeType = Node.PROCESSING_INSTRUCTION_NODE - __slots__ = ('target', 'data') - - def __init__(self, target, data): - self.target = target - self.data = data - - # nodeValue is an alias for data - def _get_nodeValue(self): - return self.data - def _set_nodeValue(self, value): - self.data = value - nodeValue = property(_get_nodeValue, _set_nodeValue) - - # nodeName is an alias for target - def _get_nodeName(self): - return self.target - def _set_nodeName(self, value): - self.target = value - nodeName = property(_get_nodeName, _set_nodeName) - - def writexml(self, writer, indent="", addindent="", newl=""): - writer.write("%s%s" % (indent,self.target, self.data, newl)) - - -class CharacterData(Childless, Node): - __slots__=('_data', 'ownerDocument','parentNode', 'previousSibling', 'nextSibling') - - def __init__(self): - self.ownerDocument = self.parentNode = None - self.previousSibling = self.nextSibling = None - self._data = '' - Node.__init__(self) - - def _get_length(self): - return len(self.data) - __len__ = _get_length - - def _get_data(self): - return self._data - def _set_data(self, data): - self._data = data - - data = nodeValue = property(_get_data, _set_data) - - def __repr__(self): - data = self.data - if len(data) > 10: - dotdotdot = "..." - else: - dotdotdot = "" - return '' % ( - self.__class__.__name__, data[0:10], dotdotdot) - - def substringData(self, offset, count): - if offset < 0: - raise xml.dom.IndexSizeErr("offset cannot be negative") - if offset >= len(self.data): - raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") - if count < 0: - raise xml.dom.IndexSizeErr("count cannot be negative") - return self.data[offset:offset+count] - - def appendData(self, arg): - self.data = self.data + arg - - def insertData(self, offset, arg): - if offset < 0: - raise xml.dom.IndexSizeErr("offset cannot be negative") - if offset >= len(self.data): - raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") - if arg: - self.data = "%s%s%s" % ( - self.data[:offset], arg, self.data[offset:]) - - def deleteData(self, offset, count): - if offset < 0: - raise xml.dom.IndexSizeErr("offset cannot be negative") - if offset >= len(self.data): - raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") - if count < 0: - raise xml.dom.IndexSizeErr("count cannot be negative") - if count: - self.data = self.data[:offset] + self.data[offset+count:] - - def replaceData(self, offset, count, arg): - if offset < 0: - raise xml.dom.IndexSizeErr("offset cannot be negative") - if offset >= len(self.data): - raise xml.dom.IndexSizeErr("offset cannot be beyond end of data") - if count < 0: - raise xml.dom.IndexSizeErr("count cannot be negative") - if count: - self.data = "%s%s%s" % ( - self.data[:offset], arg, self.data[offset+count:]) - -defproperty(CharacterData, "length", doc="Length of the string data.") - - -class Text(CharacterData): - __slots__ = () - - nodeType = Node.TEXT_NODE - nodeName = "#text" - attributes = None - - def splitText(self, offset): - if offset < 0 or offset > len(self.data): - raise xml.dom.IndexSizeErr("illegal offset value") - newText = self.__class__() - newText.data = self.data[offset:] - newText.ownerDocument = self.ownerDocument - next = self.nextSibling - if self.parentNode and self in self.parentNode.childNodes: - if next is None: - self.parentNode.appendChild(newText) - else: - self.parentNode.insertBefore(newText, next) - self.data = self.data[:offset] - return newText - - def writexml(self, writer, indent="", addindent="", newl=""): - _write_data(writer, "%s%s%s" % (indent, self.data, newl)) - - # DOM Level 3 (WD 9 April 2002) - - def _get_wholeText(self): - L = [self.data] - n = self.previousSibling - while n is not None: - if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): - L.insert(0, n.data) - n = n.previousSibling - else: - break - n = self.nextSibling - while n is not None: - if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): - L.append(n.data) - n = n.nextSibling - else: - break - return ''.join(L) - - def replaceWholeText(self, content): - # XXX This needs to be seriously changed if minidom ever - # supports EntityReference nodes. - parent = self.parentNode - n = self.previousSibling - while n is not None: - if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): - next = n.previousSibling - parent.removeChild(n) - n = next - else: - break - n = self.nextSibling - if not content: - parent.removeChild(self) - while n is not None: - if n.nodeType in (Node.TEXT_NODE, Node.CDATA_SECTION_NODE): - next = n.nextSibling - parent.removeChild(n) - n = next - else: - break - if content: - self.data = content - return self - else: - return None - - def _get_isWhitespaceInElementContent(self): - if self.data.strip(): - return False - elem = _get_containing_element(self) - if elem is None: - return False - info = self.ownerDocument._get_elem_info(elem) - if info is None: - return False - else: - return info.isElementContent() - -defproperty(Text, "isWhitespaceInElementContent", - doc="True iff this text node contains only whitespace" - " and is in element content.") -defproperty(Text, "wholeText", - doc="The text of all logically-adjacent text nodes.") - - -def _get_containing_element(node): - c = node.parentNode - while c is not None: - if c.nodeType == Node.ELEMENT_NODE: - return c - c = c.parentNode - return None - -def _get_containing_entref(node): - c = node.parentNode - while c is not None: - if c.nodeType == Node.ENTITY_REFERENCE_NODE: - return c - c = c.parentNode - return None - - -class Comment(CharacterData): - nodeType = Node.COMMENT_NODE - nodeName = "#comment" - - def __init__(self, data): - CharacterData.__init__(self) - self._data = data - - def writexml(self, writer, indent="", addindent="", newl=""): - if "--" in self.data: - raise ValueError("'--' is not allowed in a comment node") - writer.write("%s%s" % (indent, self.data, newl)) - - -class CDATASection(Text): - __slots__ = () - - nodeType = Node.CDATA_SECTION_NODE - nodeName = "#cdata-section" - - def writexml(self, writer, indent="", addindent="", newl=""): - if self.data.find("]]>") >= 0: - raise ValueError("']]>' not allowed in a CDATA section") - writer.write("" % self.data) - - -class ReadOnlySequentialNamedNodeMap(object): - __slots__ = '_seq', - - def __init__(self, seq=()): - # seq should be a list or tuple - self._seq = seq - - def __len__(self): - return len(self._seq) - - def _get_length(self): - return len(self._seq) - - def getNamedItem(self, name): - for n in self._seq: - if n.nodeName == name: - return n - - def getNamedItemNS(self, namespaceURI, localName): - for n in self._seq: - if n.namespaceURI == namespaceURI and n.localName == localName: - return n - - def __getitem__(self, name_or_tuple): - if isinstance(name_or_tuple, tuple): - node = self.getNamedItemNS(*name_or_tuple) - else: - node = self.getNamedItem(name_or_tuple) - if node is None: - raise KeyError(name_or_tuple) - return node - - def item(self, index): - if index < 0: - return None - try: - return self._seq[index] - except IndexError: - return None - - def removeNamedItem(self, name): - raise xml.dom.NoModificationAllowedErr( - "NamedNodeMap instance is read-only") - - def removeNamedItemNS(self, namespaceURI, localName): - raise xml.dom.NoModificationAllowedErr( - "NamedNodeMap instance is read-only") - - def setNamedItem(self, node): - raise xml.dom.NoModificationAllowedErr( - "NamedNodeMap instance is read-only") - - def setNamedItemNS(self, node): - raise xml.dom.NoModificationAllowedErr( - "NamedNodeMap instance is read-only") - - def __getstate__(self): - return [self._seq] - - def __setstate__(self, state): - self._seq = state[0] - -defproperty(ReadOnlySequentialNamedNodeMap, "length", - doc="Number of entries in the NamedNodeMap.") - - -class Identified: - """Mix-in class that supports the publicId and systemId attributes.""" - - __slots__ = 'publicId', 'systemId' - - def _identified_mixin_init(self, publicId, systemId): - self.publicId = publicId - self.systemId = systemId - - def _get_publicId(self): - return self.publicId - - def _get_systemId(self): - return self.systemId - -class DocumentType(Identified, Childless, Node): - nodeType = Node.DOCUMENT_TYPE_NODE - nodeValue = None - name = None - publicId = None - systemId = None - internalSubset = None - - def __init__(self, qualifiedName): - self.entities = ReadOnlySequentialNamedNodeMap() - self.notations = ReadOnlySequentialNamedNodeMap() - if qualifiedName: - prefix, localname = _nssplit(qualifiedName) - self.name = localname - self.nodeName = self.name - - def _get_internalSubset(self): - return self.internalSubset - - def cloneNode(self, deep): - if self.ownerDocument is None: - # it's ok - clone = DocumentType(None) - clone.name = self.name - clone.nodeName = self.name - operation = xml.dom.UserDataHandler.NODE_CLONED - if deep: - clone.entities._seq = [] - clone.notations._seq = [] - for n in self.notations._seq: - notation = Notation(n.nodeName, n.publicId, n.systemId) - clone.notations._seq.append(notation) - n._call_user_data_handler(operation, n, notation) - for e in self.entities._seq: - entity = Entity(e.nodeName, e.publicId, e.systemId, - e.notationName) - entity.actualEncoding = e.actualEncoding - entity.encoding = e.encoding - entity.version = e.version - clone.entities._seq.append(entity) - e._call_user_data_handler(operation, e, entity) - self._call_user_data_handler(operation, self, clone) - return clone - else: - return None - - def writexml(self, writer, indent="", addindent="", newl=""): - writer.write(""+newl) - -class Entity(Identified, Node): - attributes = None - nodeType = Node.ENTITY_NODE - nodeValue = None - - actualEncoding = None - encoding = None - version = None - - def __init__(self, name, publicId, systemId, notation): - self.nodeName = name - self.notationName = notation - self.childNodes = NodeList() - self._identified_mixin_init(publicId, systemId) - - def _get_actualEncoding(self): - return self.actualEncoding - - def _get_encoding(self): - return self.encoding - - def _get_version(self): - return self.version - - def appendChild(self, newChild): - raise xml.dom.HierarchyRequestErr( - "cannot append children to an entity node") - - def insertBefore(self, newChild, refChild): - raise xml.dom.HierarchyRequestErr( - "cannot insert children below an entity node") - - def removeChild(self, oldChild): - raise xml.dom.HierarchyRequestErr( - "cannot remove children from an entity node") - - def replaceChild(self, newChild, oldChild): - raise xml.dom.HierarchyRequestErr( - "cannot replace children of an entity node") - -class Notation(Identified, Childless, Node): - nodeType = Node.NOTATION_NODE - nodeValue = None - - def __init__(self, name, publicId, systemId): - self.nodeName = name - self._identified_mixin_init(publicId, systemId) - - -class DOMImplementation(DOMImplementationLS): - _features = [("core", "1.0"), - ("core", "2.0"), - ("core", None), - ("xml", "1.0"), - ("xml", "2.0"), - ("xml", None), - ("ls-load", "3.0"), - ("ls-load", None), - ] - - def hasFeature(self, feature, version): - if version == "": - version = None - return (feature.lower(), version) in self._features - - def createDocument(self, namespaceURI, qualifiedName, doctype): - if doctype and doctype.parentNode is not None: - raise xml.dom.WrongDocumentErr( - "doctype object owned by another DOM tree") - doc = self._create_document() - - add_root_element = not (namespaceURI is None - and qualifiedName is None - and doctype is None) - - if not qualifiedName and add_root_element: - # The spec is unclear what to raise here; SyntaxErr - # would be the other obvious candidate. Since Xerces raises - # InvalidCharacterErr, and since SyntaxErr is not listed - # for createDocument, that seems to be the better choice. - # XXX: need to check for illegal characters here and in - # createElement. - - # DOM Level III clears this up when talking about the return value - # of this function. If namespaceURI, qName and DocType are - # Null the document is returned without a document element - # Otherwise if doctype or namespaceURI are not None - # Then we go back to the above problem - raise xml.dom.InvalidCharacterErr("Element with no name") - - if add_root_element: - prefix, localname = _nssplit(qualifiedName) - if prefix == "xml" \ - and namespaceURI != "http://www.w3.org/XML/1998/namespace": - raise xml.dom.NamespaceErr("illegal use of 'xml' prefix") - if prefix and not namespaceURI: - raise xml.dom.NamespaceErr( - "illegal use of prefix without namespaces") - element = doc.createElementNS(namespaceURI, qualifiedName) - if doctype: - doc.appendChild(doctype) - doc.appendChild(element) - - if doctype: - doctype.parentNode = doctype.ownerDocument = doc - - doc.doctype = doctype - doc.implementation = self - return doc - - def createDocumentType(self, qualifiedName, publicId, systemId): - doctype = DocumentType(qualifiedName) - doctype.publicId = publicId - doctype.systemId = systemId - return doctype - - # DOM Level 3 (WD 9 April 2002) - - def getInterface(self, feature): - if self.hasFeature(feature, None): - return self - else: - return None - - # internal - def _create_document(self): - return Document() - -class ElementInfo(object): - """Object that represents content-model information for an element. - - This implementation is not expected to be used in practice; DOM - builders should provide implementations which do the right thing - using information available to it. - - """ - - __slots__ = 'tagName', - - def __init__(self, name): - self.tagName = name - - def getAttributeType(self, aname): - return _no_type - - def getAttributeTypeNS(self, namespaceURI, localName): - return _no_type - - def isElementContent(self): - return False - - def isEmpty(self): - """Returns true iff this element is declared to have an EMPTY - content model.""" - return False - - def isId(self, aname): - """Returns true iff the named attribute is a DTD-style ID.""" - return False - - def isIdNS(self, namespaceURI, localName): - """Returns true iff the identified attribute is a DTD-style ID.""" - return False - - def __getstate__(self): - return self.tagName - - def __setstate__(self, state): - self.tagName = state - -def _clear_id_cache(node): - if node.nodeType == Node.DOCUMENT_NODE: - node._id_cache.clear() - node._id_search_stack = None - elif _in_document(node): - node.ownerDocument._id_cache.clear() - node.ownerDocument._id_search_stack= None - -class Document(Node, DocumentLS): - __slots__ = ('_elem_info', 'doctype', - '_id_search_stack', 'childNodes', '_id_cache') - _child_node_types = (Node.ELEMENT_NODE, Node.PROCESSING_INSTRUCTION_NODE, - Node.COMMENT_NODE, Node.DOCUMENT_TYPE_NODE) - - implementation = DOMImplementation() - nodeType = Node.DOCUMENT_NODE - nodeName = "#document" - nodeValue = None - attributes = None - parentNode = None - previousSibling = nextSibling = None - - - # Document attributes from Level 3 (WD 9 April 2002) - - actualEncoding = None - encoding = None - standalone = None - version = None - strictErrorChecking = False - errorHandler = None - documentURI = None - - _magic_id_count = 0 - - def __init__(self): - self.doctype = None - self.childNodes = NodeList() - # mapping of (namespaceURI, localName) -> ElementInfo - # and tagName -> ElementInfo - self._elem_info = {} - self._id_cache = {} - self._id_search_stack = None - - def _get_elem_info(self, element): - if element.namespaceURI: - key = element.namespaceURI, element.localName - else: - key = element.tagName - return self._elem_info.get(key) - - def _get_actualEncoding(self): - return self.actualEncoding - - def _get_doctype(self): - return self.doctype - - def _get_documentURI(self): - return self.documentURI - - def _get_encoding(self): - return self.encoding - - def _get_errorHandler(self): - return self.errorHandler - - def _get_standalone(self): - return self.standalone - - def _get_strictErrorChecking(self): - return self.strictErrorChecking - - def _get_version(self): - return self.version - - def appendChild(self, node): - if node.nodeType not in self._child_node_types: - raise xml.dom.HierarchyRequestErr( - "%s cannot be child of %s" % (repr(node), repr(self))) - if node.parentNode is not None: - # This needs to be done before the next test since this - # may *be* the document element, in which case it should - # end up re-ordered to the end. - node.parentNode.removeChild(node) - - if node.nodeType == Node.ELEMENT_NODE \ - and self._get_documentElement(): - raise xml.dom.HierarchyRequestErr( - "two document elements disallowed") - return Node.appendChild(self, node) - - def removeChild(self, oldChild): - try: - self.childNodes.remove(oldChild) - except ValueError: - raise xml.dom.NotFoundErr() - oldChild.nextSibling = oldChild.previousSibling = None - oldChild.parentNode = None - if self.documentElement is oldChild: - self.documentElement = None - - return oldChild - - def _get_documentElement(self): - for node in self.childNodes: - if node.nodeType == Node.ELEMENT_NODE: - return node - - def unlink(self): - if self.doctype is not None: - self.doctype.unlink() - self.doctype = None - Node.unlink(self) - - def cloneNode(self, deep): - if not deep: - return None - clone = self.implementation.createDocument(None, None, None) - clone.encoding = self.encoding - clone.standalone = self.standalone - clone.version = self.version - for n in self.childNodes: - childclone = _clone_node(n, deep, clone) - assert childclone.ownerDocument.isSameNode(clone) - clone.childNodes.append(childclone) - if childclone.nodeType == Node.DOCUMENT_NODE: - assert clone.documentElement is None - elif childclone.nodeType == Node.DOCUMENT_TYPE_NODE: - assert clone.doctype is None - clone.doctype = childclone - childclone.parentNode = clone - self._call_user_data_handler(xml.dom.UserDataHandler.NODE_CLONED, - self, clone) - return clone - - def createDocumentFragment(self): - d = DocumentFragment() - d.ownerDocument = self - return d - - def createElement(self, tagName): - e = Element(tagName) - e.ownerDocument = self - return e - - def createTextNode(self, data): - if not isinstance(data, str): - raise TypeError("node contents must be a string") - t = Text() - t.data = data - t.ownerDocument = self - return t - - def createCDATASection(self, data): - if not isinstance(data, str): - raise TypeError("node contents must be a string") - c = CDATASection() - c.data = data - c.ownerDocument = self - return c - - def createComment(self, data): - c = Comment(data) - c.ownerDocument = self - return c - - def createProcessingInstruction(self, target, data): - p = ProcessingInstruction(target, data) - p.ownerDocument = self - return p - - def createAttribute(self, qName): - a = Attr(qName) - a.ownerDocument = self - a.value = "" - return a - - def createElementNS(self, namespaceURI, qualifiedName): - prefix, localName = _nssplit(qualifiedName) - e = Element(qualifiedName, namespaceURI, prefix) - e.ownerDocument = self - return e - - def createAttributeNS(self, namespaceURI, qualifiedName): - prefix, localName = _nssplit(qualifiedName) - a = Attr(qualifiedName, namespaceURI, localName, prefix) - a.ownerDocument = self - a.value = "" - return a - - # A couple of implementation-specific helpers to create node types - # not supported by the W3C DOM specs: - - def _create_entity(self, name, publicId, systemId, notationName): - e = Entity(name, publicId, systemId, notationName) - e.ownerDocument = self - return e - - def _create_notation(self, name, publicId, systemId): - n = Notation(name, publicId, systemId) - n.ownerDocument = self - return n - - def getElementById(self, id): - if id in self._id_cache: - return self._id_cache[id] - if not (self._elem_info or self._magic_id_count): - return None - - stack = self._id_search_stack - if stack is None: - # we never searched before, or the cache has been cleared - stack = [self.documentElement] - self._id_search_stack = stack - elif not stack: - # Previous search was completed and cache is still valid; - # no matching node. - return None - - result = None - while stack: - node = stack.pop() - # add child elements to stack for continued searching - stack.extend([child for child in node.childNodes - if child.nodeType in _nodeTypes_with_children]) - # check this node - info = self._get_elem_info(node) - if info: - # We have to process all ID attributes before - # returning in order to get all the attributes set to - # be IDs using Element.setIdAttribute*(). - for attr in node.attributes.values(): - if attr.namespaceURI: - if info.isIdNS(attr.namespaceURI, attr.localName): - self._id_cache[attr.value] = node - if attr.value == id: - result = node - elif not node._magic_id_nodes: - break - elif info.isId(attr.name): - self._id_cache[attr.value] = node - if attr.value == id: - result = node - elif not node._magic_id_nodes: - break - elif attr._is_id: - self._id_cache[attr.value] = node - if attr.value == id: - result = node - elif node._magic_id_nodes == 1: - break - elif node._magic_id_nodes: - for attr in node.attributes.values(): - if attr._is_id: - self._id_cache[attr.value] = node - if attr.value == id: - result = node - if result is not None: - break - return result - - def getElementsByTagName(self, name): - return _get_elements_by_tagName_helper(self, name, NodeList()) - - def getElementsByTagNameNS(self, namespaceURI, localName): - return _get_elements_by_tagName_ns_helper( - self, namespaceURI, localName, NodeList()) - - def isSupported(self, feature, version): - return self.implementation.hasFeature(feature, version) - - def importNode(self, node, deep): - if node.nodeType == Node.DOCUMENT_NODE: - raise xml.dom.NotSupportedErr("cannot import document nodes") - elif node.nodeType == Node.DOCUMENT_TYPE_NODE: - raise xml.dom.NotSupportedErr("cannot import document type nodes") - return _clone_node(node, deep, self) - - def writexml(self, writer, indent="", addindent="", newl="", encoding=None): - if encoding is None: - writer.write(''+newl) - else: - writer.write('%s' % ( - encoding, newl)) - for node in self.childNodes: - node.writexml(writer, indent, addindent, newl) - - # DOM Level 3 (WD 9 April 2002) - - def renameNode(self, n, namespaceURI, name): - if n.ownerDocument is not self: - raise xml.dom.WrongDocumentErr( - "cannot rename nodes from other documents;\n" - "expected %s,\nfound %s" % (self, n.ownerDocument)) - if n.nodeType not in (Node.ELEMENT_NODE, Node.ATTRIBUTE_NODE): - raise xml.dom.NotSupportedErr( - "renameNode() only applies to element and attribute nodes") - if namespaceURI != EMPTY_NAMESPACE: - if ':' in name: - prefix, localName = name.split(':', 1) - if ( prefix == "xmlns" - and namespaceURI != xml.dom.XMLNS_NAMESPACE): - raise xml.dom.NamespaceErr( - "illegal use of 'xmlns' prefix") - else: - if ( name == "xmlns" - and namespaceURI != xml.dom.XMLNS_NAMESPACE - and n.nodeType == Node.ATTRIBUTE_NODE): - raise xml.dom.NamespaceErr( - "illegal use of the 'xmlns' attribute") - prefix = None - localName = name - else: - prefix = None - localName = None - if n.nodeType == Node.ATTRIBUTE_NODE: - element = n.ownerElement - if element is not None: - is_id = n._is_id - element.removeAttributeNode(n) - else: - element = None - n.prefix = prefix - n._localName = localName - n.namespaceURI = namespaceURI - n.nodeName = name - if n.nodeType == Node.ELEMENT_NODE: - n.tagName = name - else: - # attribute node - n.name = name - if element is not None: - element.setAttributeNode(n) - if is_id: - element.setIdAttributeNode(n) - # It's not clear from a semantic perspective whether we should - # call the user data handlers for the NODE_RENAMED event since - # we're re-using the existing node. The draft spec has been - # interpreted as meaning "no, don't call the handler unless a - # new node is created." - return n - -defproperty(Document, "documentElement", - doc="Top-level element of this document.") - - -def _clone_node(node, deep, newOwnerDocument): - """ - Clone a node and give it the new owner document. - Called by Node.cloneNode and Document.importNode - """ - if node.ownerDocument.isSameNode(newOwnerDocument): - operation = xml.dom.UserDataHandler.NODE_CLONED - else: - operation = xml.dom.UserDataHandler.NODE_IMPORTED - if node.nodeType == Node.ELEMENT_NODE: - clone = newOwnerDocument.createElementNS(node.namespaceURI, - node.nodeName) - for attr in node.attributes.values(): - clone.setAttributeNS(attr.namespaceURI, attr.nodeName, attr.value) - a = clone.getAttributeNodeNS(attr.namespaceURI, attr.localName) - a.specified = attr.specified - - if deep: - for child in node.childNodes: - c = _clone_node(child, deep, newOwnerDocument) - clone.appendChild(c) - - elif node.nodeType == Node.DOCUMENT_FRAGMENT_NODE: - clone = newOwnerDocument.createDocumentFragment() - if deep: - for child in node.childNodes: - c = _clone_node(child, deep, newOwnerDocument) - clone.appendChild(c) - - elif node.nodeType == Node.TEXT_NODE: - clone = newOwnerDocument.createTextNode(node.data) - elif node.nodeType == Node.CDATA_SECTION_NODE: - clone = newOwnerDocument.createCDATASection(node.data) - elif node.nodeType == Node.PROCESSING_INSTRUCTION_NODE: - clone = newOwnerDocument.createProcessingInstruction(node.target, - node.data) - elif node.nodeType == Node.COMMENT_NODE: - clone = newOwnerDocument.createComment(node.data) - elif node.nodeType == Node.ATTRIBUTE_NODE: - clone = newOwnerDocument.createAttributeNS(node.namespaceURI, - node.nodeName) - clone.specified = True - clone.value = node.value - elif node.nodeType == Node.DOCUMENT_TYPE_NODE: - assert node.ownerDocument is not newOwnerDocument - operation = xml.dom.UserDataHandler.NODE_IMPORTED - clone = newOwnerDocument.implementation.createDocumentType( - node.name, node.publicId, node.systemId) - clone.ownerDocument = newOwnerDocument - if deep: - clone.entities._seq = [] - clone.notations._seq = [] - for n in node.notations._seq: - notation = Notation(n.nodeName, n.publicId, n.systemId) - notation.ownerDocument = newOwnerDocument - clone.notations._seq.append(notation) - if hasattr(n, '_call_user_data_handler'): - n._call_user_data_handler(operation, n, notation) - for e in node.entities._seq: - entity = Entity(e.nodeName, e.publicId, e.systemId, - e.notationName) - entity.actualEncoding = e.actualEncoding - entity.encoding = e.encoding - entity.version = e.version - entity.ownerDocument = newOwnerDocument - clone.entities._seq.append(entity) - if hasattr(e, '_call_user_data_handler'): - e._call_user_data_handler(operation, e, entity) - else: - # Note the cloning of Document and DocumentType nodes is - # implementation specific. minidom handles those cases - # directly in the cloneNode() methods. - raise xml.dom.NotSupportedErr("Cannot clone node %s" % repr(node)) - - # Check for _call_user_data_handler() since this could conceivably - # used with other DOM implementations (one of the FourThought - # DOMs, perhaps?). - if hasattr(node, '_call_user_data_handler'): - node._call_user_data_handler(operation, node, clone) - return clone - - -def _nssplit(qualifiedName): - fields = qualifiedName.split(':', 1) - if len(fields) == 2: - return fields - else: - return (None, fields[0]) - - -def _do_pulldom_parse(func, args, kwargs): - events = func(*args, **kwargs) - toktype, rootNode = events.getEvent() - events.expandNode(rootNode) - events.clear() - return rootNode - -def parse(file, parser=None, bufsize=None): - """Parse a file into a DOM by filename or file object.""" - if parser is None and not bufsize: - from xml.dom import expatbuilder - return expatbuilder.parse(file) - else: - from xml.dom import pulldom - return _do_pulldom_parse(pulldom.parse, (file,), - {'parser': parser, 'bufsize': bufsize}) - -def parseString(string, parser=None): - """Parse a file into a DOM from a string.""" - if parser is None: - from xml.dom import expatbuilder - return expatbuilder.parseString(string) - else: - from xml.dom import pulldom - return _do_pulldom_parse(pulldom.parseString, (string,), - {'parser': parser}) - -def getDOMImplementation(features=None): - if features: - if isinstance(features, str): - features = domreg._parse_feature_string(features) - for f, v in features: - if not Document.implementation.hasFeature(f, v): - return None - return Document.implementation